{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extending LSTMs: LSTMs with Peepholes and GRUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import collections\n",
    "import math\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from matplotlib import pylab\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "import tensorflow as tf\n",
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading Stories\n",
    "Stories are automatically downloaded from https://www.cs.cmu.edu/~spok/grimmtmp/, if not detected in the disk. The total size of stories is around ~500KB. The dataset consists of 100 stories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading file:  stories\\001.txt\n",
      "File  001.txt  already exists.\n",
      "Downloading file:  stories\\002.txt\n",
      "File  002.txt  already exists.\n",
      "Downloading file:  stories\\003.txt\n",
      "File  003.txt  already exists.\n",
      "Downloading file:  stories\\004.txt\n",
      "File  004.txt  already exists.\n",
      "Downloading file:  stories\\005.txt\n",
      "File  005.txt  already exists.\n",
      "Downloading file:  stories\\006.txt\n",
      "File  006.txt  already exists.\n",
      "Downloading file:  stories\\007.txt\n",
      "File  007.txt  already exists.\n",
      "Downloading file:  stories\\008.txt\n",
      "File  008.txt  already exists.\n",
      "Downloading file:  stories\\009.txt\n",
      "File  009.txt  already exists.\n",
      "Downloading file:  stories\\010.txt\n",
      "File  010.txt  already exists.\n",
      "Downloading file:  stories\\011.txt\n",
      "File  011.txt  already exists.\n",
      "Downloading file:  stories\\012.txt\n",
      "File  012.txt  already exists.\n",
      "Downloading file:  stories\\013.txt\n",
      "File  013.txt  already exists.\n",
      "Downloading file:  stories\\014.txt\n",
      "File  014.txt  already exists.\n",
      "Downloading file:  stories\\015.txt\n",
      "File  015.txt  already exists.\n",
      "Downloading file:  stories\\016.txt\n",
      "File  016.txt  already exists.\n",
      "Downloading file:  stories\\017.txt\n",
      "File  017.txt  already exists.\n",
      "Downloading file:  stories\\018.txt\n",
      "File  018.txt  already exists.\n",
      "Downloading file:  stories\\019.txt\n",
      "File  019.txt  already exists.\n",
      "Downloading file:  stories\\020.txt\n",
      "File  020.txt  already exists.\n",
      "Downloading file:  stories\\021.txt\n",
      "File  021.txt  already exists.\n",
      "Downloading file:  stories\\022.txt\n",
      "File  022.txt  already exists.\n",
      "Downloading file:  stories\\023.txt\n",
      "File  023.txt  already exists.\n",
      "Downloading file:  stories\\024.txt\n",
      "File  024.txt  already exists.\n",
      "Downloading file:  stories\\025.txt\n",
      "File  025.txt  already exists.\n",
      "Downloading file:  stories\\026.txt\n",
      "File  026.txt  already exists.\n",
      "Downloading file:  stories\\027.txt\n",
      "File  027.txt  already exists.\n",
      "Downloading file:  stories\\028.txt\n",
      "File  028.txt  already exists.\n",
      "Downloading file:  stories\\029.txt\n",
      "File  029.txt  already exists.\n",
      "Downloading file:  stories\\030.txt\n",
      "File  030.txt  already exists.\n",
      "Downloading file:  stories\\031.txt\n",
      "File  031.txt  already exists.\n",
      "Downloading file:  stories\\032.txt\n",
      "File  032.txt  already exists.\n",
      "Downloading file:  stories\\033.txt\n",
      "File  033.txt  already exists.\n",
      "Downloading file:  stories\\034.txt\n",
      "File  034.txt  already exists.\n",
      "Downloading file:  stories\\035.txt\n",
      "File  035.txt  already exists.\n",
      "Downloading file:  stories\\036.txt\n",
      "File  036.txt  already exists.\n",
      "Downloading file:  stories\\037.txt\n",
      "File  037.txt  already exists.\n",
      "Downloading file:  stories\\038.txt\n",
      "File  038.txt  already exists.\n",
      "Downloading file:  stories\\039.txt\n",
      "File  039.txt  already exists.\n",
      "Downloading file:  stories\\040.txt\n",
      "File  040.txt  already exists.\n",
      "Downloading file:  stories\\041.txt\n",
      "File  041.txt  already exists.\n",
      "Downloading file:  stories\\042.txt\n",
      "File  042.txt  already exists.\n",
      "Downloading file:  stories\\043.txt\n",
      "File  043.txt  already exists.\n",
      "Downloading file:  stories\\044.txt\n",
      "File  044.txt  already exists.\n",
      "Downloading file:  stories\\045.txt\n",
      "File  045.txt  already exists.\n",
      "Downloading file:  stories\\046.txt\n",
      "File  046.txt  already exists.\n",
      "Downloading file:  stories\\047.txt\n",
      "File  047.txt  already exists.\n",
      "Downloading file:  stories\\048.txt\n",
      "File  048.txt  already exists.\n",
      "Downloading file:  stories\\049.txt\n",
      "File  049.txt  already exists.\n",
      "Downloading file:  stories\\050.txt\n",
      "File  050.txt  already exists.\n",
      "Downloading file:  stories\\051.txt\n",
      "File  051.txt  already exists.\n",
      "Downloading file:  stories\\052.txt\n",
      "File  052.txt  already exists.\n",
      "Downloading file:  stories\\053.txt\n",
      "File  053.txt  already exists.\n",
      "Downloading file:  stories\\054.txt\n",
      "File  054.txt  already exists.\n",
      "Downloading file:  stories\\055.txt\n",
      "File  055.txt  already exists.\n",
      "Downloading file:  stories\\056.txt\n",
      "File  056.txt  already exists.\n",
      "Downloading file:  stories\\057.txt\n",
      "File  057.txt  already exists.\n",
      "Downloading file:  stories\\058.txt\n",
      "File  058.txt  already exists.\n",
      "Downloading file:  stories\\059.txt\n",
      "File  059.txt  already exists.\n",
      "Downloading file:  stories\\060.txt\n",
      "File  060.txt  already exists.\n",
      "Downloading file:  stories\\061.txt\n",
      "File  061.txt  already exists.\n",
      "Downloading file:  stories\\062.txt\n",
      "File  062.txt  already exists.\n",
      "Downloading file:  stories\\063.txt\n",
      "File  063.txt  already exists.\n",
      "Downloading file:  stories\\064.txt\n",
      "File  064.txt  already exists.\n",
      "Downloading file:  stories\\065.txt\n",
      "File  065.txt  already exists.\n",
      "Downloading file:  stories\\066.txt\n",
      "File  066.txt  already exists.\n",
      "Downloading file:  stories\\067.txt\n",
      "File  067.txt  already exists.\n",
      "Downloading file:  stories\\068.txt\n",
      "File  068.txt  already exists.\n",
      "Downloading file:  stories\\069.txt\n",
      "File  069.txt  already exists.\n",
      "Downloading file:  stories\\070.txt\n",
      "File  070.txt  already exists.\n",
      "Downloading file:  stories\\071.txt\n",
      "File  071.txt  already exists.\n",
      "Downloading file:  stories\\072.txt\n",
      "File  072.txt  already exists.\n",
      "Downloading file:  stories\\073.txt\n",
      "File  073.txt  already exists.\n",
      "Downloading file:  stories\\074.txt\n",
      "File  074.txt  already exists.\n",
      "Downloading file:  stories\\075.txt\n",
      "File  075.txt  already exists.\n",
      "Downloading file:  stories\\076.txt\n",
      "File  076.txt  already exists.\n",
      "Downloading file:  stories\\077.txt\n",
      "File  077.txt  already exists.\n",
      "Downloading file:  stories\\078.txt\n",
      "File  078.txt  already exists.\n",
      "Downloading file:  stories\\079.txt\n",
      "File  079.txt  already exists.\n",
      "Downloading file:  stories\\080.txt\n",
      "File  080.txt  already exists.\n",
      "Downloading file:  stories\\081.txt\n",
      "File  081.txt  already exists.\n",
      "Downloading file:  stories\\082.txt\n",
      "File  082.txt  already exists.\n",
      "Downloading file:  stories\\083.txt\n",
      "File  083.txt  already exists.\n",
      "Downloading file:  stories\\084.txt\n",
      "File  084.txt  already exists.\n",
      "Downloading file:  stories\\085.txt\n",
      "File  085.txt  already exists.\n",
      "Downloading file:  stories\\086.txt\n",
      "File  086.txt  already exists.\n",
      "Downloading file:  stories\\087.txt\n",
      "File  087.txt  already exists.\n",
      "Downloading file:  stories\\088.txt\n",
      "File  088.txt  already exists.\n",
      "Downloading file:  stories\\089.txt\n",
      "File  089.txt  already exists.\n",
      "Downloading file:  stories\\090.txt\n",
      "File  090.txt  already exists.\n",
      "Downloading file:  stories\\091.txt\n",
      "File  091.txt  already exists.\n",
      "Downloading file:  stories\\092.txt\n",
      "File  092.txt  already exists.\n",
      "Downloading file:  stories\\093.txt\n",
      "File  093.txt  already exists.\n",
      "Downloading file:  stories\\094.txt\n",
      "File  094.txt  already exists.\n",
      "Downloading file:  stories\\095.txt\n",
      "File  095.txt  already exists.\n",
      "Downloading file:  stories\\096.txt\n",
      "File  096.txt  already exists.\n",
      "Downloading file:  stories\\097.txt\n",
      "File  097.txt  already exists.\n",
      "Downloading file:  stories\\098.txt\n",
      "File  098.txt  already exists.\n",
      "Downloading file:  stories\\099.txt\n",
      "File  099.txt  already exists.\n",
      "Downloading file:  stories\\100.txt\n",
      "File  100.txt  already exists.\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.cs.cmu.edu/~spok/grimmtmp/'\n",
    "\n",
    "# Create a directory if needed\n",
    "dir_name = 'stories'\n",
    "if not os.path.exists(dir_name):\n",
    "    os.mkdir(dir_name)\n",
    "    \n",
    "def maybe_download(filename):\n",
    "  \"\"\"Download a file if not present\"\"\"\n",
    "  print('Downloading file: ', dir_name+ os.sep+filename)\n",
    "    \n",
    "  if not os.path.exists(dir_name+os.sep+filename):\n",
    "    filename, _ = urlretrieve(url + filename, dir_name+os.sep+filename)\n",
    "  else:\n",
    "    print('File ',filename, ' already exists.')\n",
    "  \n",
    "  return filename\n",
    "\n",
    "num_files = 100\n",
    "filenames = [format(i, '03d')+'.txt' for i in range(1,num_files+1)]\n",
    "\n",
    "for fn in filenames:\n",
    "    maybe_download(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 files found.\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(filenames)):\n",
    "    file_exists = os.path.isfile(os.path.join(dir_name,filenames[i]))\n",
    "    assert file_exists\n",
    "print('%d files found.'%len(filenames))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading data\n",
    "Data will be stored in a list of lists where the each list represents a document and document is a list of words. We will then break the text into bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file stories\\001.txt\n",
      "Data size (Characters) (Document 0) 3667\n",
      "Sample string (Document 0) ['in', ' o', 'ld', 'en', ' t', 'im', 'es', ' w', 'he', 'n ', 'wi', 'sh', 'in', 'g ', 'st', 'il', 'l ', 'he', 'lp', 'ed', ' o', 'ne', ', ', 'th', 'er', 'e ', 'li', 've', 'd ', 'a ', 'ki', 'ng', '\\nw', 'ho', 'se', ' d', 'au', 'gh', 'te', 'rs', ' w', 'er', 'e ', 'al', 'l ', 'be', 'au', 'ti', 'fu', 'l,']\n",
      "\n",
      "Processing file stories\\002.txt\n",
      "Data size (Characters) (Document 1) 4928\n",
      "Sample string (Document 1) ['ha', 'rd', ' b', 'y ', 'a ', 'gr', 'ea', 't ', 'fo', 're', 'st', ' d', 'we', 'lt', ' a', ' w', 'oo', 'd-', 'cu', 'tt', 'er', ' w', 'it', 'h ', 'hi', 's ', 'wi', 'fe', ', ', 'wh', 'o ', 'ha', 'd ', 'an', '\\no', 'nl', 'y ', 'ch', 'il', 'd,', ' a', ' l', 'it', 'tl', 'e ', 'gi', 'rl', ' t', 'hr', 'ee']\n",
      "\n",
      "Processing file stories\\003.txt\n",
      "Data size (Characters) (Document 2) 9745\n",
      "Sample string (Document 2) ['a ', 'ce', 'rt', 'ai', 'n ', 'fa', 'th', 'er', ' h', 'ad', ' t', 'wo', ' s', 'on', 's,', ' t', 'he', ' e', 'ld', 'er', ' o', 'f ', 'wh', 'om', ' w', 'as', ' s', 'ma', 'rt', ' a', 'nd', '\\ns', 'en', 'si', 'bl', 'e,', ' a', 'nd', ' c', 'ou', 'ld', ' d', 'o ', 'ev', 'er', 'yt', 'hi', 'ng', ', ', 'bu']\n",
      "\n",
      "Processing file stories\\004.txt\n",
      "Data size (Characters) (Document 3) 2852\n",
      "Sample string (Document 3) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', 'n ', 'ol', 'd ', 'go', 'at', ' w', 'ho', ' h', 'ad', ' s', 'ev', 'en', ' l', 'it', 'tl', 'e ', 'ki', 'ds', ', ', 'an', 'd\\n', 'lo', 've', 'd ', 'th', 'em', ' w', 'it', 'h ', 'al', 'l ', 'th', 'e ', 'lo', 've', ' o']\n",
      "\n",
      "Processing file stories\\005.txt\n",
      "Data size (Characters) (Document 4) 8189\n",
      "Sample string (Document 4) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', 'n ', 'ol', 'd ', 'ki', 'ng', ' w', 'ho', ' w', 'as', ' i', 'll', ' a', 'nd', ' t', 'ho', 'ug', 'ht', ' t', 'o\\n', 'hi', 'ms', 'el', 'f ', \"'i\", ' a', 'm ', 'ly', 'in', 'g ', 'on', ' w', 'ha', 't ', 'mu', 'st', ' b']\n",
      "\n",
      "Processing file stories\\006.txt\n",
      "Data size (Characters) (Document 5) 4369\n",
      "Sample string (Document 5) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'ea', 'sa', 'nt', ' w', 'ho', ' h', 'ad', ' d', 'ri', 've', 'n ', 'hi', 's ', 'co', 'w ', 'to', ' t', 'he', ' f', 'ai', 'r,', ' a', 'nd', ' s', 'ol', 'd\\n', 'he', 'r ', 'fo', 'r ', 'se', 've', 'n ', 'ta', 'le', 'rs', '. ', ' o', 'n ', 'th', 'e ']\n",
      "\n",
      "Processing file stories\\007.txt\n",
      "Data size (Characters) (Document 6) 5216\n",
      "Sample string (Document 6) ['th', 'er', 'e ', 'we', 're', ' o', 'nc', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'a ', 'ki', 'ng', ' a', 'nd', ' a', ' q', 'ue', 'en', ' w', 'ho', ' l', 'iv', 'ed', '\\nh', 'ap', 'pi', 'ly', ' t', 'og', 'et', 'he', 'r ', 'an', 'd ', 'ha', 'd ', 'tw', 'el', 've', ' c', 'hi', 'ld', 're', 'n,', ' b']\n",
      "\n",
      "Processing file stories\\008.txt\n",
      "Data size (Characters) (Document 7) 6097\n",
      "Sample string (Document 7) ['li', 'tt', 'le', ' b', 'ro', 'th', 'er', ' t', 'oo', 'k ', 'hi', 's ', 'li', 'tt', 'le', ' s', 'is', 'te', 'r ', 'by', ' t', 'he', ' h', 'an', 'd ', 'an', 'd ', 'sa', 'id', ', ', 'si', 'nc', 'e\\n', 'ou', 'r ', 'mo', 'th', 'er', ' d', 'ie', 'd ', 'we', ' h', 'av', 'e ', 'ha', 'd ', 'no', ' h', 'ap']\n",
      "\n",
      "Processing file stories\\009.txt\n",
      "Data size (Characters) (Document 8) 3699\n",
      "Sample string (Document 8) ['th', 'er', 'e ', 'we', 're', ' o', 'nc', 'e ', 'a ', 'ma', 'n ', 'an', 'd ', 'a ', 'wo', 'ma', 'n ', 'wh', 'o ', 'ha', 'd ', 'lo', 'ng', ' i', 'n ', 'va', 'in', '\\nw', 'is', 'he', 'd ', 'fo', 'r ', 'a ', 'ch', 'il', 'd.', '  ', 'at', ' l', 'en', 'gt', 'h ', 'th', 'e ', 'wo', 'ma', 'n ', 'ho', 'pe']\n",
      "\n",
      "Processing file stories\\010.txt\n",
      "Data size (Characters) (Document 9) 5268\n",
      "Sample string (Document 9) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', 'se', ' w', 'if', 'e ', 'di', 'ed', ', ', 'an', 'd ', 'a ', 'wo', 'ma', 'n ', 'wh', 'os', 'e ', 'hu', 'sb', 'an', 'd\\n', 'di', 'ed', ', ', 'an', 'd ', 'th', 'e ', 'ma', 'n ', 'ha', 'd ', 'a ', 'da', 'ug', 'ht', 'er', ', ', 'an']\n",
      "\n",
      "Processing file stories\\011.txt\n",
      "Data size (Characters) (Document 10) 2377\n",
      "Sample string (Document 10) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' g', 'ir', 'l ', 'wh', 'o ', 'wa', 's ', 'id', 'le', ' a', 'nd', ' w', 'ou', 'ld', ' n', 'ot', ' s', 'pi', 'n,', ' a', 'nd', '\\nl', 'et', ' h', 'er', ' m', 'ot', 'he', 'r ', 'sa', 'y ', 'wh', 'at', ' s', 'he', ' w', 'ou', 'ld', ', ', 'sh', 'e ', 'co']\n",
      "\n",
      "Processing file stories\\012.txt\n",
      "Data size (Characters) (Document 11) 7695\n",
      "Sample string (Document 11) ['ha', 'rd', ' b', 'y ', 'a ', 'gr', 'ea', 't ', 'fo', 're', 'st', ' d', 'we', 'lt', ' a', ' p', 'oo', 'r ', 'wo', 'od', '-c', 'ut', 'te', 'r ', 'wi', 'th', ' h', 'is', ' w', 'if', 'e\\n', 'an', 'd ', 'hi', 's ', 'tw', 'o ', 'ch', 'il', 'dr', 'en', '. ', ' t', 'he', ' b', 'oy', ' w', 'as', ' c', 'al']\n",
      "\n",
      "Processing file stories\\013.txt\n",
      "Data size (Characters) (Document 12) 3665\n",
      "Sample string (Document 12) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' o', 'n ', 'a ', 'ti', 'me', ' a', ' p', 'oo', 'r ', 'ma', 'n,', ' w', 'ho', ' c', 'ou', 'ld', ' n', 'o ', 'lo', 'ng', 'er', '\\ns', 'up', 'po', 'rt', ' h', 'is', ' o', 'nl', 'y ', 'so', 'n.', '  ', 'th', 'en', ' s', 'ai', 'd ', 'th', 'e ', 'so', 'n,', ' d']\n",
      "\n",
      "Processing file stories\\014.txt\n",
      "Data size (Characters) (Document 13) 4178\n",
      "Sample string (Document 13) ['a ', 'lo', 'ng', ' t', 'im', 'e ', 'ag', 'o ', 'th', 'er', 'e ', 'li', 've', 'd ', 'a ', 'ki', 'ng', ' w', 'ho', ' w', 'as', ' f', 'am', 'ed', ' f', 'or', ' h', 'is', ' w', 'is', 'do', 'm\\n', 'th', 'ro', 'ug', 'h ', 'al', 'l ', 'th', 'e ', 'la', 'nd', '. ', ' n', 'ot', 'hi', 'ng', ' w', 'as', ' h']\n",
      "\n",
      "Processing file stories\\015.txt\n",
      "Data size (Characters) (Document 14) 8674\n",
      "Sample string (Document 14) ['on', 'e ', 'su', 'mm', 'er', \"'s\", ' m', 'or', 'ni', 'ng', ' a', ' l', 'it', 'tl', 'e ', 'ta', 'il', 'or', ' w', 'as', ' s', 'it', 'ti', 'ng', ' o', 'n ', 'hi', 's ', 'ta', 'bl', 'e\\n', 'by', ' t', 'he', ' w', 'in', 'do', 'w,', ' h', 'e ', 'wa', 's ', 'in', ' g', 'oo', 'd ', 'sp', 'ir', 'it', 's,']\n",
      "\n",
      "Processing file stories\\016.txt\n",
      "Data size (Characters) (Document 15) 7018\n",
      "Sample string (Document 15) ['\\tc', 'in', 'de', 're', 'll', 'a\\n', 'th', 'e ', 'wi', 'fe', ' o', 'f ', 'a ', 'ri', 'ch', ' m', 'an', ' f', 'el', 'l ', 'si', 'ck', ', ', 'an', 'd ', 'as', ' s', 'he', ' f', 'el', 't ', 'th', 'at', ' h', 'er', ' e', 'nd', '\\nw', 'as', ' d', 'ra', 'wi', 'ng', ' n', 'ea', 'r,', ' s', 'he', ' c', 'al']\n",
      "\n",
      "Processing file stories\\017.txt\n",
      "Data size (Characters) (Document 16) 3039\n",
      "Sample string (Document 16) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' k', 'in', \"g'\", 's ', 'so', 'n ', 'wh', 'o ', 'wa', 's ', 'se', 'iz', 'ed', ' w', 'it', 'h ', 'a ', 'de', 'si', 're', ' t', 'o ', 'tr', 'av', 'el', '\\na', 'bo', 'ut', ' t', 'he', ' w', 'or', 'ld', ', ', 'an', 'd ', 'to', 'ok', ' n', 'o ', 'on', 'e ']\n",
      "\n",
      "Processing file stories\\018.txt\n",
      "Data size (Characters) (Document 17) 3020\n",
      "Sample string (Document 17) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' w', 'id', 'ow', ' w', 'ho', ' h', 'ad', ' t', 'wo', ' d', 'au', 'gh', 'te', 'rs', ' -', ' o', 'ne', ' o', 'f\\n', 'wh', 'om', ' w', 'as', ' p', 're', 'tt', 'y ', 'an', 'd ', 'in', 'du', 'st', 'ri', 'ou', 's,', ' w', 'hi', 'ls', 't ', 'th', 'e ', 'ot']\n",
      "\n",
      "Processing file stories\\019.txt\n",
      "Data size (Characters) (Document 18) 2465\n",
      "Sample string (Document 18) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', ' h', 'ad', ' s', 'ev', 'en', ' s', 'on', 's,', ' a', 'nd', ' s', 'ti', 'll', ' h', 'e ', 'ha', 'd\\n', 'no', ' d', 'au', 'gh', 'te', 'r,', ' h', 'ow', 'ev', 'er', ' m', 'uc', 'h ', 'he', ' w', 'is', 'he', 'd ', 'fo', 'r ', 'on']\n",
      "\n",
      "Processing file stories\\020.txt\n",
      "Data size (Characters) (Document 19) 3703\n",
      "Sample string (Document 19) ['\\tl', 'it', 'tl', 'e ', 're', 'd-', 'ca', 'p\\n', '\\no', 'nc', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'th', 'er', 'e ', 'wa', 's ', 'a ', 'de', 'ar', ' l', 'it', 'tl', 'e ', 'gi', 'rl', ' w', 'ho', ' w', 'as', ' l', 'ov', 'ed', '\\nb', 'y ', 'ev', 'er', 'y ', 'on', 'e ', 'wh', 'o ', 'lo', 'ok', 'ed']\n",
      "\n",
      "Processing file stories\\021.txt\n",
      "Data size (Characters) (Document 20) 1924\n",
      "Sample string (Document 20) ['in', ' a', ' c', 'er', 'ta', 'in', ' c', 'ou', 'nt', 'ry', ' t', 'he', 're', ' w', 'as', ' o', 'nc', 'e ', 'gr', 'ea', 't ', 'la', 'me', 'nt', 'at', 'io', 'n ', 'ov', 'er', ' a', '\\nw', 'il', 'd ', 'bo', 'ar', ' t', 'ha', 't ', 'la', 'id', ' w', 'as', 'te', ' t', 'he', ' f', 'ar', 'me', \"r'\", 's ']\n",
      "\n",
      "Processing file stories\\022.txt\n",
      "Data size (Characters) (Document 21) 6561\n",
      "Sample string (Document 21) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'wo', 'ma', 'n ', 'wh', 'o ', 'ga', 've', ' b', 'ir', 'th', ' t', 'o ', 'a ', 'li', 'tt', 'le', ' s', 'on', ',\\n', 'an', 'd ', 'as', ' h', 'e ', 'ca', 'me', ' i', 'nt', 'o ', 'th', 'e ', 'wo', 'rl', 'd ', 'wi', 'th', ' a', ' c', 'au']\n",
      "\n",
      "Processing file stories\\023.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 22) 5956\n",
      "Sample string (Document 22) ['a ', 'ce', 'rt', 'ai', 'n ', 'mi', 'll', 'er', ' h', 'ad', ' l', 'it', 'tl', 'e ', 'by', ' l', 'it', 'tl', 'e ', 'fa', 'll', 'en', ' i', 'nt', 'o ', 'po', 've', 'rt', 'y,', ' a', 'nd', '\\nh', 'ad', ' n', 'ot', 'hi', 'ng', ' l', 'ef', 't ', 'bu', 't ', 'hi', 's ', 'mi', 'll', ' a', 'nd', ' a', ' l']\n",
      "\n",
      "Processing file stories\\024.txt\n",
      "Data size (Characters) (Document 23) 2529\n",
      "Sample string (Document 23) ['th', 'e ', 'mo', 'th', 'er', ' o', 'f ', 'ha', 'ns', ' s', 'ai', 'd,', ' w', 'hi', 'th', 'er', ' a', 'wa', 'y,', ' h', 'an', 's.', '  ', 'ha', 'ns', ' a', 'ns', 'we', 're', 'd,', ' t', 'o\\n', 'gr', 'et', 'el', '. ', ' b', 'eh', 'av', 'e ', 'we', 'll', ', ', 'ha', 'ns', '. ', ' o', 'h,', ' i', \"'l\"]\n",
      "\n",
      "Processing file stories\\025.txt\n",
      "Data size (Characters) (Document 24) 2416\n",
      "Sample string (Document 24) ['an', ' a', 'ge', 'd ', 'co', 'un', 't ', 'on', 'ce', ' l', 'iv', 'ed', ' i', 'n ', 'sw', 'it', 'ze', 'rl', 'an', 'd,', ' w', 'ho', ' h', 'ad', ' a', 'n ', 'on', 'ly', ' s', 'on', ',\\n', 'bu', 't ', 'he', ' w', 'as', ' s', 'tu', 'pi', 'd,', ' a', 'nd', ' c', 'ou', 'ld', ' l', 'ea', 'rn', ' n', 'ot']\n",
      "\n",
      "Processing file stories\\026.txt\n",
      "Data size (Characters) (Document 25) 3369\n",
      "Sample string (Document 25) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', ' h', 'ad', ' a', ' d', 'au', 'gh', 'te', 'r ', 'wh', 'o ', 'wa', 's ', 'ca', 'll', 'ed', ' c', 'le', 've', 'r\\n', 'el', 'si', 'e.', '  ', 'an', 'd ', 'wh', 'en', ' s', 'he', ' h', 'ad', ' g', 'ro', 'wn', ' u', 'p ', 'he', 'r ']\n",
      "\n",
      "Processing file stories\\027.txt\n",
      "Data size (Characters) (Document 26) 10013\n",
      "Sample string (Document 26) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' t', 'ai', 'lo', 'r ', 'wh', 'o ', 'ha', 'd ', 'th', 're', 'e ', 'so', 'ns', ', ', 'an', 'd\\n', 'on', 'ly', ' o', 'ne', ' g', 'oa', 't.', '  ', 'bu', 't ', 'as', ' t', 'he', ' g', 'oa', 't ', 'su', 'pp', 'or', 'te']\n",
      "\n",
      "Processing file stories\\028.txt\n",
      "Data size (Characters) (Document 27) 5788\n",
      "Sample string (Document 27) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'pe', 'as', 'an', 't ', 'wh', 'o ', 'sa', 't ', 'in', ' t', 'he', ' e', 've', 'ni', 'ng', ' b', 'y ', 'th', 'e\\n', 'he', 'ar', 'th', ' a', 'nd', ' p', 'ok', 'ed', ' t', 'he', ' f', 'ir', 'e,', ' a', 'nd', ' h', 'is', ' w', 'if', 'e ']\n",
      "\n",
      "Processing file stories\\029.txt\n",
      "Data size (Characters) (Document 28) 1335\n",
      "Sample string (Document 28) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'se', 'rv', 'an', 't-', 'gi', 'rl', ' w', 'ho', ' w', 'as', ' i', 'nd', 'us', 'tr', 'io', 'us', ' a', 'nd', ' c', 'le', 'an', 'ly', '\\na', 'nd', ' s', 'we', 'pt', ' t', 'he', ' h', 'ou', 'se', ' e', 've', 'ry', ' d', 'ay', ', ', 'an']\n",
      "\n",
      "Processing file stories\\030.txt\n",
      "Data size (Characters) (Document 29) 3591\n",
      "Sample string (Document 29) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' m', 'il', 'le', 'r,', ' w', 'ho', ' h', 'ad', ' a', ' b', 'ea', 'ut', 'if', 'ul', '\\nd', 'au', 'gh', 'te', 'r,', ' a', 'nd', ' a', 's ', 'sh', 'e ', 'wa', 's ', 'gr', 'ow', 'n ', 'up', ', ', 'he', ' w', 'is', 'he']\n",
      "\n",
      "Processing file stories\\031.txt\n",
      "Data size (Characters) (Document 30) 1624\n",
      "Sample string (Document 30) ['a ', 'po', 'or', ' m', 'an', ' h', 'ad', ' s', 'o ', 'ma', 'ny', ' c', 'hi', 'ld', 're', 'n ', 'th', 'at', ' h', 'e ', 'ha', 'd ', 'al', 're', 'ad', 'y ', 'as', 'ke', 'd\\n', 'ev', 'er', 'yo', 'ne', ' i', 'n ', 'th', 'e ', 'wo', 'rl', 'd ', 'to', ' b', 'e ', 'go', 'df', 'at', 'he', 'r,', ' a', 'nd']\n",
      "\n",
      "Processing file stories\\032.txt\n",
      "Data size (Characters) (Document 31) 758\n",
      "Sample string (Document 31) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' l', 'it', 'tl', 'e ', 'gi', 'rl', ' w', 'ho', ' w', 'as', ' o', 'bs', 'ti', 'na', 'te', ' a', 'nd', ' i', 'nq', 'ui', 'si', 'ti', 've', ',\\n', 'an', 'd ', 'wh', 'en', ' h', 'er', ' p', 'ar', 'en', 'ts', ' t', 'ol', 'd ', 'he', 'r ', 'to', ' d', 'o ']\n",
      "\n",
      "Processing file stories\\033.txt\n",
      "Data size (Characters) (Document 32) 3121\n",
      "Sample string (Document 32) ['a ', 'po', 'or', ' m', 'an', ' h', 'ad', ' t', 'we', 'lv', 'e ', 'ch', 'il', 'dr', 'en', ' a', 'nd', ' w', 'as', ' f', 'or', 'ce', 'd ', 'to', ' w', 'or', 'k ', 'ni', 'gh', 't ', 'an', 'd\\n', 'da', 'y ', 'to', ' g', 'iv', 'e ', 'th', 'em', ' e', 've', 'n ', 'br', 'ea', 'd.', '  ', 'wh', 'en', ' t']\n",
      "\n",
      "Processing file stories\\034.txt\n",
      "Data size (Characters) (Document 33) 4192\n",
      "Sample string (Document 33) ['a ', 'ce', 'rt', 'ai', 'n ', 'ta', 'il', 'or', ' h', 'ad', ' a', ' s', 'on', ', ', 'wh', 'o ', 'ha', 'pp', 'en', 'ed', ' t', 'o ', 'be', ' s', 'ma', 'll', ', ', 'an', 'd\\n', 'no', ' b', 'ig', 'ge', 'r ', 'th', 'an', ' a', ' t', 'hu', 'mb', ', ', 'an', 'd ', 'on', ' t', 'hi', 's ', 'ac', 'co', 'un']\n",
      "\n",
      "Processing file stories\\035.txt\n",
      "Data size (Characters) (Document 34) 3650\n",
      "Sample string (Document 34) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' w', 'iz', 'ar', 'd ', 'wh', 'o ', 'us', 'ed', ' t', 'o ', 'ta', 'ke', ' t', 'he', ' f', 'or', 'm ', 'of', ' a', ' p', 'oo', 'r\\n', 'ma', 'n,', ' a', 'nd', ' w', 'en', 't ', 'to', ' h', 'ou', 'se', 's ', 'an', 'd ', 'be', 'gg', 'ed', ', ', 'an', 'd ']\n",
      "\n",
      "Processing file stories\\036.txt\n",
      "Data size (Characters) (Document 35) 8219\n",
      "Sample string (Document 35) ['it', ' i', 's ', 'no', 'w ', 'lo', 'ng', ' a', 'go', ', ', 'qu', 'it', 'e ', 'tw', 'o ', 'th', 'ou', 'sa', 'nd', ' y', 'ea', 'rs', ', ', 'si', 'nc', 'e ', 'th', 'er', 'e ', 'wa', 's\\n', 'a ', 'ri', 'ch', ' m', 'an', ' w', 'ho', ' h', 'ad', ' a', ' b', 'ea', 'ut', 'if', 'ul', ' a', 'nd', ' p', 'io']\n",
      "\n",
      "Processing file stories\\037.txt\n",
      "Data size (Characters) (Document 36) 2151\n",
      "Sample string (Document 36) ['a ', 'fa', 'rm', 'er', ' o', 'nc', 'e ', 'ha', 'd ', 'a ', 'fa', 'it', 'hf', 'ul', ' d', 'og', ' c', 'al', 'le', 'd ', 'su', 'lt', 'an', ', ', 'wh', 'o ', 'ha', 'd ', 'gr', 'ow', 'n\\n', 'ol', 'd,', ' a', 'nd', ' l', 'os', 't ', 'al', 'l ', 'hi', 's ', 'te', 'et', 'h,', ' s', 'o ', 'th', 'at', ' h']\n",
      "\n",
      "Processing file stories\\038.txt\n",
      "Data size (Characters) (Document 37) 5129\n",
      "Sample string (Document 37) ['on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ', ', 'a ', 'ce', 'rt', 'ai', 'n ', 'ki', 'ng', ' w', 'as', ' h', 'un', 'ti', 'ng', ' i', 'n ', 'a ', 'gr', 'ea', 't ', 'fo', 're', 'st', ',\\n', 'an', 'd ', 'he', ' c', 'ha', 'se', 'd ', 'a ', 'wi', 'ld', ' b', 'ea', 'st', ' s', 'o ', 'ea', 'ge', 'rl']\n",
      "\n",
      "Processing file stories\\039.txt\n",
      "Data size (Characters) (Document 38) 3472\n",
      "Sample string (Document 38) ['\\tb', 'ri', 'ar', '-r', 'os', 'e\\n', '\\na', ' l', 'on', 'g ', 'ti', 'me', ' a', 'go', ' t', 'he', 're', ' w', 'er', 'e ', 'a ', 'ki', 'ng', ' a', 'nd', ' q', 'ue', 'en', ' w', 'ho', ' s', 'ai', 'd ', 'ev', 'er', 'y\\n', 'da', 'y,', ' a', 'h,', ' i', 'f ', 'on', 'ly', ' w', 'e ', 'ha', 'd ', 'a ', 'ch']\n",
      "\n",
      "Processing file stories\\040.txt\n",
      "Data size (Characters) (Document 39) 2490\n",
      "Sample string (Document 39) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' f', 'or', 'es', 'te', 'r ', 'wh', 'o ', 'we', 'nt', ' i', 'nt', 'o ', 'th', 'e ', 'fo', 're', 'st', ' t', 'o ', 'hu', 'nt', ',\\n', 'an', 'd ', 'as', ' h', 'e ', 'en', 'te', 're', 'd ', 'it', ' h', 'e ', 'he', 'ar', 'd ', 'a ', 'so', 'un', 'd ', 'of']\n",
      "\n",
      "Processing file stories\\041.txt\n",
      "Data size (Characters) (Document 40) 4273\n",
      "Sample string (Document 40) ['a ', 'ki', 'ng', ' h', 'ad', ' a', ' d', 'au', 'gh', 'te', 'r ', 'wh', 'o ', 'wa', 's ', 'be', 'au', 'ti', 'fu', 'l ', 'be', 'yo', 'nd', ' a', 'll', ' m', 'ea', 'su', 're', ',\\n', 'bu', 't ', 'so', ' p', 'ro', 'ud', ' a', 'nd', ' h', 'au', 'gh', 'ty', ' w', 'it', 'ha', 'l ', 'th', 'at', ' n', 'o ']\n",
      "\n",
      "Processing file stories\\042.txt\n",
      "Data size (Characters) (Document 41) 8327\n",
      "Sample string (Document 41) ['\\ts', 'no', 'w ', 'wh', 'it', 'e ', 'an', 'd ', 'th', 'e ', 'se', 've', 'n ', 'dw', 'ar', 'fs', '\\n\\n', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' i', 'n ', 'th', 'e ', 'mi', 'dd', 'le', ' o', 'f ', 'wi', 'nt', 'er', ', ', 'wh', 'en', ' t', 'he', ' f', 'la', 'ke', 's ', 'of', '\\ns', 'no', 'w ']\n",
      "\n",
      "Processing file stories\\043.txt\n",
      "Data size (Characters) (Document 42) 6128\n",
      "Sample string (Document 42) ['th', 'er', 'e ', 'we', 're', ' o', 'nc', 'e ', 'th', 're', 'e ', 'br', 'ot', 'he', 'rs', ' w', 'ho', ' h', 'ad', ' f', 'al', 'le', 'n ', 'de', 'ep', 'er', ' a', 'nd', ' d', 'ee', 'pe', 'r ', 'in', 'to', '\\np', 'ov', 'er', 'ty', ', ', 'an', 'd ', 'at', ' l', 'as', 't ', 'th', 'ei', 'r ', 'ne', 'ed']\n",
      "\n",
      "Processing file stories\\044.txt\n",
      "Data size (Characters) (Document 43) 2819\n",
      "Sample string (Document 43) ['\\tr', 'um', 'pe', 'ls', 'ti', 'lt', 'sk', 'in', '\\n\\n', 'on', 'ce', ' t', 'he', 're', ' w', 'as', ' a', ' m', 'il', 'le', 'r ', 'wh', 'o ', 'wa', 's ', 'po', 'or', ', ', 'bu', 't ', 'wh', 'o ', 'ha', 'd ', 'a ', 'be', 'au', 'ti', 'fu', 'l\\n', 'da', 'ug', 'ht', 'er', '. ', ' n', 'ow', ' i', 't ', 'ha']\n",
      "\n",
      "Processing file stories\\045.txt\n",
      "Data size (Characters) (Document 44) 3822\n",
      "Sample string (Document 44) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' w', 'om', 'an', ' w', 'ho', ' w', 'as', ' a', ' r', 'ea', 'l ', 'wi', 'tc', 'h ', 'an', 'd ', 'ha', 'd ', 'tw', 'o\\n', 'da', 'ug', 'ht', 'er', 's,', ' o', 'ne', ' u', 'gl', 'y ', 'an', 'd ', 'wi', 'ck', 'ed', ', ']\n",
      "\n",
      "Processing file stories\\046.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 45) 7772\n",
      "Sample string (Document 45) ['in', ' o', 'ld', 'en', ' t', 'im', 'es', ' t', 'he', 're', ' w', 'as', ' a', ' k', 'in', 'g,', ' w', 'ho', ' h', 'ad', ' b', 'eh', 'in', 'd ', 'hi', 's ', 'pa', 'la', 'ce', ' a', '\\nb', 'ea', 'ut', 'if', 'ul', ' p', 'le', 'as', 'ur', 'e-', 'ga', 'rd', 'en', ' i', 'n ', 'wh', 'ic', 'h ', 'th', 'er']\n",
      "\n",
      "Processing file stories\\047.txt\n",
      "Data size (Characters) (Document 46) 22158\n",
      "Sample string (Document 46) ['th', 'er', 'e ', 'we', 're', ' o', 'nc', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'tw', 'o ', 'br', 'ot', 'he', 'rs', ', ', 'on', 'e ', 'ri', 'ch', ' a', 'nd', ' t', 'he', ' o', 'th', 'er', '\\np', 'oo', 'r.', '  ', 'th', 'e ', 'ri', 'ch', ' o', 'ne', ' w', 'as', ' a', ' g', 'ol', 'ds', 'mi', 'th']\n",
      "\n",
      "Processing file stories\\048.txt\n",
      "Data size (Characters) (Document 47) 2169\n",
      "Sample string (Document 47) ['tw', 'o ', 'ki', 'ng', \"s'\", ' s', 'on', 's ', 'on', 'ce', ' w', 'en', 't ', 'ou', 't ', 'in', ' s', 'ea', 'rc', 'h ', 'of', ' a', 'dv', 'en', 'tu', 're', 's,', ' a', 'nd', ' f', 'el', 'l ', 'in', 'to', '\\na', ' w', 'il', 'd,', ' d', 'is', 'or', 'de', 'rl', 'y ', 'wa', 'y ', 'of', ' l', 'iv', 'in']\n",
      "\n",
      "Processing file stories\\049.txt\n",
      "Data size (Characters) (Document 48) 2822\n",
      "Sample string (Document 48) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' k', 'in', 'g ', 'wh', 'o ', 'ha', 'd ', 'th', 're', 'e ', 'so', 'ns', ', ', 'of', ' w', 'ho', 'm ', 'tw', 'o\\n', 'we', 're', ' c', 'le', 've', 'r ', 'an', 'd ', 'wi', 'se', ', ', 'bu', 't ', 'th', 'e ', 'th', 'ir']\n",
      "\n",
      "Processing file stories\\050.txt\n",
      "Data size (Characters) (Document 49) 4034\n",
      "Sample string (Document 49) ['th', 'er', 'e ', 'wa', 's ', 'a ', 'ma', 'n ', 'wh', 'o ', 'ha', 'd ', 'th', 're', 'e ', 'so', 'ns', ', ', 'th', 'e ', 'yo', 'un', 'ge', 'st', ' o', 'f ', 'wh', 'om', ' w', 'as', ' c', 'al', 'le', 'd\\n', 'du', 'mm', 'li', 'ng', ', ', 'an', 'd ', 'wa', 's ', 'de', 'sp', 'is', 'ed', ', ', 'mo', 'ck']\n",
      "\n",
      "Processing file stories\\051.txt\n",
      "Data size (Characters) (Document 50) 5608\n",
      "Sample string (Document 50) ['\\ta', 'll', 'er', 'le', 'ir', 'au', 'h\\n', '\\nt', 'he', 're', ' w', 'as', ' o', 'nc', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'a ', 'ki', 'ng', ' w', 'ho', ' h', 'ad', ' a', ' w', 'if', 'e ', 'wi', 'th', ' g', 'ol', 'de', 'n ', 'ha', 'ir', ',\\n', 'an', 'd ', 'sh', 'e ', 'wa', 's ', 'so', ' b', 'ea']\n",
      "\n",
      "Processing file stories\\052.txt\n",
      "Data size (Characters) (Document 51) 1287\n",
      "Sample string (Document 51) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' w', 'om', 'an', ' a', 'nd', ' h', 'er', ' d', 'au', 'gh', 'te', 'r ', 'wh', 'o ', 'li', 've', 'd ', 'in', ' a', '\\np', 're', 'tt', 'y ', 'ga', 'rd', 'en', ' w', 'it', 'h ', 'ca', 'bb', 'ag', 'es', '. ', ' a', 'nd', ' a', ' l', 'it', 'tl', 'e ', 'ha']\n",
      "\n",
      "Processing file stories\\053.txt\n",
      "Data size (Characters) (Document 52) 2841\n",
      "Sample string (Document 52) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' k', 'in', \"g'\", 's ', 'so', 'n ', 'wh', 'o ', 'ha', 'd ', 'a ', 'br', 'id', 'e ', 'wh', 'om', ' h', 'e ', 'lo', 've', 'd ', 've', 'ry', ' m', 'uc', 'h.', '\\na', 'nd', ' w', 'he', 'n ', 'he', ' w', 'as', ' s', 'it', 'ti', 'ng', ' b', 'es', 'id', 'e ']\n",
      "\n",
      "Processing file stories\\054.txt\n",
      "Data size (Characters) (Document 53) 1922\n",
      "Sample string (Document 53) ['ha', 'ns', ' w', 'is', 'he', 'd ', 'to', ' p', 'ut', ' h', 'is', ' s', 'on', ' t', 'o ', 'le', 'ar', 'n ', 'a ', 'tr', 'ad', 'e,', ' s', 'o ', 'he', ' w', 'en', 't ', 'in', 'to', ' t', 'he', '\\nc', 'hu', 'rc', 'h ', 'an', 'd ', 'pr', 'ay', 'ed', ' t', 'o ', 'ou', 'r ', 'lo', 'rd', ' g', 'od', ' t']\n",
      "\n",
      "Processing file stories\\055.txt\n",
      "Data size (Characters) (Document 54) 2573\n",
      "Sample string (Document 54) ['a ', 'fa', 'th', 'er', ' o', 'nc', 'e ', 'ca', 'll', 'ed', ' h', 'is', ' t', 'hr', 'ee', ' s', 'on', 's ', 'be', 'fo', 're', ' h', 'im', ', ', 'an', 'd ', 'he', ' g', 'av', 'e ', 'to', ' t', 'he', '\\nf', 'ir', 'st', ' a', ' c', 'oc', 'k,', ' t', 'o ', 'th', 'e ', 'se', 'co', 'nd', ' a', ' s', 'cy']\n",
      "\n",
      "Processing file stories\\056.txt\n",
      "Data size (Characters) (Document 55) 5285\n",
      "Sample string (Document 55) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', ' u', 'nd', 'er', 'st', 'oo', 'd ', 'al', 'l ', 'ki', 'nd', 's ', 'of', ' a', 'rt', 's.', '  ', 'he', ' s', 'er', 've', 'd ', 'in', '\\nw', 'ar', ', ', 'an', 'd ', 'be', 'ha', 've', 'd ', 'we', 'll', ' a', 'nd', ' b', 'ra', 've']\n",
      "\n",
      "Processing file stories\\057.txt\n",
      "Data size (Characters) (Document 56) 971\n",
      "Sample string (Document 56) ['th', 'e ', 'sh', 'e-', 'wo', 'lf', ' b', 'ro', 'ug', 'ht', ' i', 'nt', 'o ', 'th', 'e ', 'wo', 'rl', 'd ', 'a ', 'yo', 'un', 'g ', 'on', 'e,', ' a', 'nd', ' i', 'nv', 'it', 'ed', ' t', 'he', ' f', 'ox', '\\nt', 'o ', 'be', ' g', 'od', 'fa', 'th', 'er', '. ', ' a', 'ft', 'er', ' a', 'll', ', ', 'he']\n",
      "\n",
      "Processing file stories\\058.txt\n",
      "Data size (Characters) (Document 57) 4538\n",
      "Sample string (Document 57) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' q', 'ue', 'en', ' t', 'o ', 'wh', 'om', ' g', 'od', ' h', 'ad', ' g', 'iv', 'en', ' n', 'o ', 'ch', 'il', 'dr', 'en', '.\\n', 'ev', 'er', 'y ', 'mo', 'rn', 'in', 'g ', 'sh', 'e ', 'we', 'nt', ' i', 'nt', 'o ', 'th']\n",
      "\n",
      "Processing file stories\\059.txt\n",
      "Data size (Characters) (Document 58) 636\n",
      "Sample string (Document 58) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' v', 'er', 'y ', 'ol', 'd ', 'ma', 'n,', ' w', 'ho', 'se', ' e', 'ye', 's ', 'ha', 'd ', 'be', 'co', 'me', ' d', 'im', ', ', 'hi', 's ', 'ea', 'rs', '\\nd', 'ul', 'l ', 'of', ' h', 'ea', 'ri', 'ng', ', ', 'hi', 's ', 'kn', 'ee', 's ', 'tr', 'em', 'bl']\n",
      "\n",
      "Processing file stories\\060.txt\n",
      "Data size (Characters) (Document 59) 786\n",
      "Sample string (Document 59) ['a ', 'li', 'tt', 'le', ' b', 'ro', 'th', 'er', ' a', 'nd', ' s', 'is', 'te', 'r ', 'we', 're', ' o', 'nc', 'e ', 'pl', 'ay', 'in', 'g ', 'by', ' a', ' w', 'el', 'l,', ' a', 'nd', ' w', 'hi', 'le', '\\nt', 'he', 'y ', 'we', 're', ' t', 'hu', 's ', 'pl', 'ay', 'in', 'g,', ' t', 'he', 'y ', 'bo', 'th']\n",
      "\n",
      "Processing file stories\\061.txt\n",
      "Data size (Characters) (Document 60) 10687\n",
      "Sample string (Document 60) ['th', 'er', 'e ', 'wa', 's ', 'on', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'a ', 'gr', 'ea', 't ', 'wa', 'r,', ' a', 'nd', ' w', 'he', 'n ', 'it', ' c', 'am', 'e ', 'to', ' a', 'n ', 'en', 'd,', '\\nm', 'an', 'y ', 'so', 'ld', 'ie', 'rs', ' w', 'er', 'e ', 'di', 'sc', 'ha', 'rg', 'ed', '. ', ' t']\n",
      "\n",
      "Processing file stories\\062.txt\n",
      "Data size (Characters) (Document 61) 5105\n",
      "Sample string (Document 61) ['ha', 'ns', ' h', 'ad', ' s', 'er', 've', 'd ', 'hi', 's ', 'ma', 'st', 'er', ' f', 'or', ' s', 'ev', 'en', ' y', 'ea', 'rs', ', ', 'so', ' h', 'e ', 'sa', 'id', ' t', 'o ', 'hi', 'm,', '\\nm', 'as', 'te', 'r,', ' m', 'y ', 'ti', 'me', ' i', 's ', 'up', ', ', 'no', 'w ', 'i ', 'sh', 'ou', 'ld', ' b']\n",
      "\n",
      "Processing file stories\\063.txt\n",
      "Data size (Characters) (Document 62) 1127\n",
      "Sample string (Document 62) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' y', 'ou', 'ng', ' p', 'ea', 'sa', 'nt', ' n', 'am', 'ed', ' h', 'an', 's,', ' w', 'ho', 'se', ' u', 'nc', 'le', '\\nw', 'an', 'te', 'd ', 'to', ' f', 'in', 'd ', 'hi', 'm ', 'a ', 'ri', 'ch', ' w', 'if', 'e.', '  ']\n",
      "\n",
      "Processing file stories\\064.txt\n",
      "Data size (Characters) (Document 63) 4981\n",
      "Sample string (Document 63) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'ma', 'n ', 'an', 'd ', 'a ', 'po', 'or', ' w', 'om', 'an', ' w', 'ho', ' h', 'ad', ' n', 'ot', 'hi', 'ng', ' b', 'ut', ' a', '\\nl', 'it', 'tl', 'e ', 'co', 'tt', 'ag', 'e,', ' a', 'nd', ' w', 'ho', ' e', 'ar', 'ne', 'd ', 'th', 'ei']\n",
      "\n",
      "Processing file stories\\065.txt\n",
      "Data size (Characters) (Document 64) 6006\n",
      "Sample string (Document 64) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' m', 'an', ' w', 'ho', ' w', 'as', ' a', 'bo', 'ut', ' t', 'o ', 'se', 't ', 'ou', 't ', 'on', ' a', ' l', 'on', 'g\\n', 'jo', 'ur', 'ne', 'y,', ' a', 'nd', ' o', 'n ', 'pa', 'rt', 'in', 'g ', 'he', ' a', 'sk', 'ed']\n",
      "\n",
      "Processing file stories\\066.txt\n",
      "Data size (Characters) (Document 65) 5900\n",
      "Sample string (Document 65) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', 'n ', 'ol', 'd ', 'qu', 'ee', 'n ', 'wh', 'os', 'e ', 'hu', 'sb', 'an', 'd ', 'ha', 'd ', 'be', 'en', ' d', 'ea', 'd\\n', 'fo', 'r ', 'ma', 'ny', ' y', 'ea', 'rs', ', ', 'an', 'd ', 'sh', 'e ', 'ha', 'd ', 'a ', 'be']\n",
      "\n",
      "Processing file stories\\067.txt\n",
      "Data size (Characters) (Document 66) 7837\n",
      "Sample string (Document 66) ['on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' c', 'ou', 'nt', 'ry', 'ma', 'n ', 'ha', 'd ', 'a ', 'so', 'n ', 'wh', 'o ', 'wa', 's ', 'as', ' b', 'ig', ' a', 's ', 'a ', 'th', 'um', 'b,', '\\na', 'nd', ' d', 'id', ' n', 'ot', ' b', 'ec', 'om', 'e ', 'an', 'y ', 'bi', 'gg', 'er', ', ', 'an']\n",
      "\n",
      "Processing file stories\\068.txt\n",
      "Data size (Characters) (Document 67) 4717\n",
      "Sample string (Document 67) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' r', 'ic', 'h ', 'ki', 'ng', ' w', 'ho', ' h', 'ad', ' t', 'hr', 'ee', ' d', 'au', 'gh', 'te', 'rs', ', ', 'wh', 'o\\n', 'da', 'il', 'y ', 'we', 'nt', ' t', 'o ', 'wa', 'lk', ' i', 'n ', 'th', 'e ', 'pa', 'la', 'ce']\n",
      "\n",
      "Processing file stories\\069.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 68) 6233\n",
      "Sample string (Document 68) ['th', 'er', 'e ', 'wa', 's ', 'a ', 'ce', 'rt', 'ai', 'n ', 'me', 'rc', 'ha', 'nt', ' w', 'ho', ' h', 'ad', ' t', 'wo', ' c', 'hi', 'ld', 're', 'n,', ' a', ' b', 'oy', ' a', 'nd', ' a', ' g', 'ir', 'l,', '\\nt', 'he', 'y ', 'we', 're', ' b', 'ot', 'h ', 'yo', 'un', 'g,', ' a', 'nd', ' c', 'ou', 'ld']\n",
      "\n",
      "Processing file stories\\070.txt\n",
      "Data size (Characters) (Document 69) 5664\n",
      "Sample string (Document 69) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' q', 'ue', 'en', ' w', 'ho', ' h', 'ad', ' a', ' l', 'it', 'tl', 'e ', 'da', 'ug', 'ht', 'er', ' w', 'ho', '\\nw', 'as', ' s', 'ti', 'll', ' s', 'o ', 'yo', 'un', 'g ', 'th', 'at', ' s', 'he', ' h', 'ad', ' t', 'o ']\n",
      "\n",
      "Processing file stories\\071.txt\n",
      "Data size (Characters) (Document 70) 3569\n",
      "Sample string (Document 70) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'pe', 'as', 'an', 't ', 'wh', 'o ', 'ha', 'd ', 'no', ' l', 'an', 'd,', ' b', 'ut', ' o', 'nl', 'y ', 'a ', 'sm', 'al', 'l\\n', 'ho', 'us', 'e,', ' a', 'nd', ' o', 'ne', ' d', 'au', 'gh', 'te', 'r.', '  ', 'th', 'en', ' s', 'ai', 'd ']\n",
      "\n",
      "Processing file stories\\072.txt\n",
      "Data size (Characters) (Document 71) 3793\n",
      "Sample string (Document 71) ['ab', 'ou', 't ', 'a ', 'th', 'ou', 'sa', 'nd', ' o', 'r ', 'mo', 're', ' y', 'ea', 'rs', ' a', 'go', ', ', 'th', 'er', 'e ', 'we', 're', ' i', 'n ', 'th', 'is', '\\nc', 'ou', 'nt', 'ry', ' n', 'ot', 'hi', 'ng', ' b', 'ut', ' s', 'ma', 'll', ' k', 'in', 'gs', ', ', 'an', 'd ', 'on', 'e ', 'of', ' t']\n",
      "\n",
      "Processing file stories\\073.txt\n",
      "Data size (Characters) (Document 72) 5980\n",
      "Sample string (Document 72) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' k', 'in', 'g ', 'wh', 'o ', 'ha', 'd ', 'an', ' i', 'll', 'ne', 'ss', ', ', 'an', 'd ', 'no', ' o', 'ne', ' b', 'el', 'ie', 've', 'd ', 'th', 'at', ' h', 'e\\n', 'wo', 'ul', 'd ', 'co', 'me', ' o', 'ut', ' o', 'f ', 'it', ' w', 'it', 'h ', 'hi', 's ']\n",
      "\n",
      "Processing file stories\\074.txt\n",
      "Data size (Characters) (Document 73) 4518\n",
      "Sample string (Document 73) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'wo', 'od', 'cu', 'tt', 'er', ' w', 'ho', ' t', 'oi', 'le', 'd ', 'fr', 'om', ' e', 'ar', 'ly', '\\nm', 'or', 'ni', 'ng', ' t', 'il', 'l ', 'la', 'te', ' a', 't ', 'ni', 'gh', 't.', '  ', 'wh', 'en', ' a', 't ', 'la', 'st', ' h', 'e ']\n",
      "\n",
      "Processing file stories\\075.txt\n",
      "Data size (Characters) (Document 74) 3247\n",
      "Sample string (Document 74) ['a ', 'di', 'sc', 'ha', 'rg', 'ed', ' s', 'ol', 'di', 'er', ' h', 'ad', ' n', 'ot', 'hi', 'ng', ' t', 'o ', 'li', 've', ' o', 'n,', ' a', 'nd', ' d', 'id', ' n', 'ot', ' k', 'no', 'w ', 'ho', 'w ', 'to', '\\nm', 'ak', 'e ', 'hi', 's ', 'wa', 'y.', '  ', 'so', ' h', 'e ', 'we', 'nt', ' o', 'ut', ' i']\n",
      "\n",
      "Processing file stories\\076.txt\n",
      "Data size (Characters) (Document 75) 5130\n",
      "Sample string (Document 75) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' y', 'ou', 'ng', ' f', 'el', 'lo', 'w ', 'wh', 'o ', 'en', 'li', 'st', 'ed', ' a', 's ', 'a ', 'so', 'ld', 'ie', 'r,', ' c', 'on', 'du', 'ct', 'ed', '\\nh', 'im', 'se', 'lf', ' b', 'ra', 've', 'ly', ', ', 'an', 'd ', 'wa', 's ', 'al', 'wa', 'ys', ' t']\n",
      "\n",
      "Processing file stories\\077.txt\n",
      "Data size (Characters) (Document 76) 2401\n",
      "Sample string (Document 76) ['on', 'ce', ' i', 'n ', 'su', 'mm', 'er', '-t', 'im', 'e ', 'th', 'e ', 'be', 'ar', ' a', 'nd', ' t', 'he', ' w', 'ol', 'f ', 'we', 're', ' w', 'al', 'ki', 'ng', ' i', 'n ', 'th', 'e ', 'fo', 're', 'st', ',\\n', 'an', 'd ', 'th', 'e ', 'be', 'ar', ' h', 'ea', 'rd', ' a', ' b', 'ir', 'd ', 'si', 'ng']\n",
      "\n",
      "Processing file stories\\078.txt\n",
      "Data size (Characters) (Document 77) 624\n",
      "Sample string (Document 77) ['th', 'er', 'e ', 'wa', 's ', 'a ', 'po', 'or', ' b', 'ut', ' g', 'oo', 'd ', 'li', 'tt', 'le', ' g', 'ir', 'l ', 'wh', 'o ', 'li', 've', 'd ', 'al', 'on', 'e ', 'wi', 'th', ' h', 'er', '\\nm', 'ot', 'he', 'r,', ' a', 'nd', ' t', 'he', 'y ', 'no', ' l', 'on', 'ge', 'r ', 'ha', 'd ', 'an', 'yt', 'hi']\n",
      "\n",
      "Processing file stories\\079.txt\n",
      "Data size (Characters) (Document 78) 3991\n",
      "Sample string (Document 78) ['on', 'e ', 'da', 'y ', 'a ', 'pe', 'as', 'an', 't ', 'to', 'ok', ' h', 'is', ' g', 'oo', 'd ', 'ha', 'ze', 'l-', 'st', 'ic', 'k ', 'ou', 't ', 'of', ' t', 'he', ' c', 'or', 'ne', 'r\\n', 'an', 'd ', 'sa', 'id', ' t', 'o ', 'hi', 's ', 'wi', 'fe', ', ', 'tr', 'in', 'a,', ' i', ' a', 'm ', 'go', 'in']\n",
      "\n",
      "Processing file stories\\080.txt\n",
      "Data size (Characters) (Document 79) 1426\n",
      "Sample string (Document 79) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' l', 'it', 'tl', 'e ', 'ch', 'il', 'd ', 'wh', 'os', 'e ', 'mo', 'th', 'er', ' g', 'av', 'e ', 'he', 'r ', 'ev', 'er', 'y\\n', 'af', 'te', 'rn', 'oo', 'n ', 'a ', 'sm', 'al', 'l ', 'bo', 'wl', ' o', 'f ', 'mi', 'lk', ' a', 'nd', ' b', 're', 'ad', ', ']\n",
      "\n",
      "Processing file stories\\081.txt\n",
      "Data size (Characters) (Document 80) 3574\n",
      "Sample string (Document 80) ['in', ' a', ' c', 'er', 'ta', 'in', ' m', 'il', 'l ', 'li', 've', 'd ', 'an', ' o', 'ld', ' m', 'il', 'le', 'r ', 'wh', 'o ', 'ha', 'd ', 'ne', 'it', 'he', 'r ', 'wi', 'fe', ' n', 'or', ' c', 'hi', 'ld', ',\\n', 'an', 'd ', 'th', 're', 'e ', 'ap', 'pr', 'en', 'ti', 'ce', 's ', 'se', 'rv', 'ed', ' u']\n",
      "\n",
      "Processing file stories\\082.txt\n",
      "Data size (Characters) (Document 81) 10822\n",
      "Sample string (Document 81) ['hi', 'll', ' a', 'nd', ' v', 'al', 'e ', 'do', ' n', 'ot', ' m', 'ee', 't,', ' b', 'ut', ' t', 'he', ' c', 'hi', 'ld', 're', 'n ', 'of', ' m', 'en', ' d', 'o,', ' g', 'oo', 'd ', 'an', 'd ', 'ba', 'd.', '\\ni', 'n ', 'th', 'is', ' w', 'ay', ' a', ' s', 'ho', 'em', 'ak', 'er', ' a', 'nd', ' a', ' t']\n",
      "\n",
      "Processing file stories\\083.txt\n",
      "Data size (Characters) (Document 82) 5480\n",
      "Sample string (Document 82) ['\\th', 'an', 's ', 'th', 'e ', 'he', 'dg', 'eh', 'og', '\\n\\n', 'th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' c', 'ou', 'nt', 'ry', ' m', 'an', ' w', 'ho', ' h', 'ad', ' m', 'on', 'ey', ' a', 'nd', ' l', 'an', 'd ', 'in', ' p', 'le', 'nt', 'y,', ' b', 'ut', '\\nh', 'ow', 'ev', 'er', ' r', 'ic', 'h ']\n",
      "\n",
      "Processing file stories\\084.txt\n",
      "Data size (Characters) (Document 83) 658\n",
      "Sample string (Document 83) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'ot', 'he', 'r ', 'wh', 'o ', 'ha', 'd ', 'a ', 'li', 'tt', 'le', ' b', 'oy', ' o', 'f ', 'se', 've', 'n ', 'ye', 'ar', 's ', 'ol', 'd,', ' w', 'ho', '\\nw', 'as', ' s', 'o ', 'ha', 'nd', 'so', 'me', ' a', 'nd', ' l', 'ov', 'ab', 'le', ' t', 'ha']\n",
      "\n",
      "Processing file stories\\085.txt\n",
      "Data size (Characters) (Document 84) 5989\n",
      "Sample string (Document 84) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' y', 'ou', 'ng', ' f', 'el', 'lo', 'w ', 'wh', 'o ', 'ha', 'd ', 'le', 'ar', 'nt', ' t', 'he', ' t', 'ra', 'de', ' o', 'f ', 'lo', 'ck', 'sm', 'it', 'h,', '\\na', 'nd', ' t', 'ol', 'd ', 'hi', 's ', 'fa', 'th', 'er', ' h', 'e ', 'wo', 'ul', 'd ', 'no']\n",
      "\n",
      "Processing file stories\\086.txt\n",
      "Data size (Characters) (Document 85) 8758\n",
      "Sample string (Document 85) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' k', 'in', 'g ', 'wh', 'o ', 'ha', 'd ', 'a ', 'li', 'tt', 'le', ' b', 'oy', ' i', 'n ', 'wh', 'os', 'e ', 'st', 'ar', 's\\n', 'it', ' h', 'ad', ' b', 'ee', 'n ', 'fo', 're', 'to', 'ld', ' t', 'ha', 't ', 'he', ' s']\n",
      "\n",
      "Processing file stories\\087.txt\n",
      "Data size (Characters) (Document 86) 3109\n",
      "Sample string (Document 86) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' p', 'ri', 'nc', 'es', 's ', 'wh', 'o ', 'wa', 's ', 'ex', 'tr', 'em', 'el', 'y ', 'pr', 'ou', 'd.', ' i', 'f ', 'a\\n', 'wo', 'oe', 'r ', 'ca', 'me', ' s', 'he', ' g', 'av', 'e ', 'hi', 'm ', 'so', 'me', ' r', 'id']\n",
      "\n",
      "Processing file stories\\088.txt\n",
      "Data size (Characters) (Document 87) 1365\n",
      "Sample string (Document 87) ['a ', 'ta', 'il', 'or', \"'s\", ' a', 'pp', 're', 'nt', 'ic', 'e ', 'wa', 's ', 'tr', 'av', 'el', 'in', 'g ', 'ab', 'ou', 't ', 'th', 'e ', 'wo', 'rl', 'd ', 'in', ' s', 'ea', 'rc', 'h ', 'of', '\\nw', 'or', 'k,', ' a', 'nd', ' a', 't ', 'on', 'e ', 'ti', 'me', ' h', 'e ', 'co', 'ul', 'd ', 'fi', 'nd']\n",
      "\n",
      "Processing file stories\\089.txt\n",
      "Data size (Characters) (Document 88) 4538\n",
      "Sample string (Document 88) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' o', 'n ', 'a ', 'ti', 'me', ' a', ' s', 'ol', 'di', 'er', ' w', 'ho', ' f', 'or', ' m', 'an', 'y ', 'ye', 'ar', 's ', 'ha', 'd ', 'se', 'rv', 'ed', ' t', 'he', '\\nk', 'in', 'g ', 'fa', 'it', 'hf', 'ul', 'ly', ', ', 'bu', 't ', 'wh', 'en', ' t', 'he', ' w']\n",
      "\n",
      "Processing file stories\\090.txt\n",
      "Data size (Characters) (Document 89) 345\n",
      "Sample string (Document 89) ['on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' t', 'he', 're', ' w', 'as', ' a', ' c', 'hi', 'ld', ' w', 'ho', ' w', 'as', ' w', 'il', 'lf', 'ul', ', ', 'an', 'd ', 'wo', 'ul', 'd ', 'no', 't ', 'do', '\\nw', 'ha', 't ', 'he', 'r ', 'mo', 'th', 'er', ' w', 'is', 'he', 'd.', '  ', 'fo', 'r ', 'th']\n",
      "\n",
      "Processing file stories\\091.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 90) 5460\n",
      "Sample string (Document 90) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' k', 'in', \"g'\", 's ', 'so', 'n,', ' w', 'ho', ' w', 'as', ' n', 'o ', 'lo', 'ng', 'er', ' c', 'on', 'te', 'nt', ' t', 'o ', 'st', 'ay', ' a', 't\\n', 'ho', 'me', ' i', 'n ', 'hi', 's ', 'fa', 'th', 'er', \"'s\", ' h', 'ou', 'se', ', ', 'an', 'd ', 'as']\n",
      "\n",
      "Processing file stories\\092.txt\n",
      "Data size (Characters) (Document 91) 6854\n",
      "Sample string (Document 91) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' y', 'ou', 'ng', ' h', 'un', 'ts', 'ma', 'n ', 'wh', 'o ', 'we', 'nt', ' i', 'nt', 'o ', 'th', 'e ', 'fo', 're', 'st', ' t', 'o ', 'li', 'e ', 'in', '\\nw', 'ai', 't.', '  ', 'he', ' h', 'ad', ' a', ' f', 're', 'sh', ' a', 'nd', ' j', 'oy', 'ou', 's ']\n",
      "\n",
      "Processing file stories\\093.txt\n",
      "Data size (Characters) (Document 92) 2314\n",
      "Sample string (Document 92) ['a ', 'po', 'or', ' s', 'er', 'va', 'nt', '-g', 'ir', 'l ', 'wa', 's ', 'on', 'ce', ' t', 'ra', 've', 'li', 'ng', ' w', 'it', 'h ', 'th', 'e ', 'fa', 'mi', 'ly', ' w', 'it', 'h ', 'wh', 'ic', 'h ', 'sh', 'e\\n', 'wa', 's ', 'in', ' s', 'er', 'vi', 'ce', ', ', 'th', 'ro', 'ug', 'h ', 'a ', 'gr', 'ea']\n",
      "\n",
      "Processing file stories\\094.txt\n",
      "Data size (Characters) (Document 93) 1706\n",
      "Sample string (Document 93) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', ' h', 'ad', ' t', 'hr', 'ee', ' s', 'on', 's,', ' a', 'nd', ' n', 'ot', 'hi', 'ng', ' e', 'ls', 'e ', 'in', ' t', 'he', '\\nw', 'or', 'ld', ' b', 'ut', ' t', 'he', ' h', 'ou', 'se', ' i', 'n ', 'wh', 'ic', 'h ', 'he', ' l', 'iv']\n",
      "\n",
      "Processing file stories\\095.txt\n",
      "Data size (Characters) (Document 94) 3229\n",
      "Sample string (Document 94) ['th', 'er', 'e ', 'wa', 's ', 'a ', 'gr', 'ea', 't ', 'wa', 'r,', ' a', 'nd', ' t', 'he', ' k', 'in', 'g ', 'ha', 'd ', 'ma', 'ny', ' s', 'ol', 'di', 'er', 's,', ' b', 'ut', ' g', 'av', 'e ', 'th', 'em', '\\ns', 'ma', 'll', ' p', 'ay', ', ', 'so', ' s', 'ma', 'll', ' t', 'ha', 't ', 'th', 'ey', ' c']\n",
      "\n",
      "Processing file stories\\096.txt\n",
      "Data size (Characters) (Document 95) 4954\n",
      "Sample string (Document 95) ['on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' l', 'iv', 'ed', ' a', ' m', 'an', ' a', 'nd', ' a', ' w', 'om', 'an', ' w', 'ho', ' s', 'o ', 'lo', 'ng', ' a', 's ', 'th', 'ey', ' w', 'er', 'e\\n', 'ri', 'ch', ' h', 'ad', ' n', 'o ', 'ch', 'il', 'dr', 'en', ', ', 'bu', 't ', 'wh', 'en', ' t', 'he']\n",
      "\n",
      "Processing file stories\\097.txt\n",
      "Data size (Characters) (Document 96) 5732\n",
      "Sample string (Document 96) ['in', ' t', 'he', ' d', 'ay', 's ', 'wh', 'en', ' w', 'is', 'hi', 'ng', ' w', 'as', ' s', 'ti', 'll', ' o', 'f ', 'so', 'me', ' u', 'se', ', ', 'a ', 'ki', 'ng', \"'s\", ' s', 'on', ' w', 'as', '\\nb', 'ew', 'it', 'ch', 'ed', ' b', 'y ', 'an', ' o', 'ld', ' w', 'it', 'ch', ', ', 'an', 'd ', 'sh', 'ut']\n",
      "\n",
      "Processing file stories\\098.txt\n",
      "Data size (Characters) (Document 97) 4334\n",
      "Sample string (Document 97) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'ma', 'n ', 'wh', 'o ', 'ha', 'd ', 'fo', 'ur', ' s', 'on', 's,', ' a', 'nd', ' w', 'he', 'n ', 'th', 'ey', ' w', 'er', 'e ', 'gr', 'ow', 'n\\n', 'up', ', ', 'he', ' s', 'ai', 'd ', 'to', ' t', 'he', 'm,', ' \"', 'my', ' d', 'ea', 'r ']\n",
      "\n",
      "Processing file stories\\099.txt\n",
      "Data size (Characters) (Document 98) 7090\n",
      "Sample string (Document 98) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' w', 'om', 'an', ' w', 'ho', ' h', 'ad', ' t', 'hr', 'ee', ' d', 'au', 'gh', 'te', 'rs', ', ', 'th', 'e ', 'el', 'de', 'st', ' o', 'f ', 'wh', 'om', '\\nw', 'as', ' c', 'al', 'le', 'd ', 'on', 'e-', 'ey', 'e,', ' b', 'ec', 'au', 'se', ' s', 'he', ' h']\n",
      "\n",
      "Processing file stories\\100.txt\n",
      "Data size (Characters) (Document 99) 1007\n",
      "Sample string (Document 99) ['\"g', 'oo', 'd-', 'da', 'y,', ' f', 'at', 'he', 'r ', 'ho', 'll', 'en', 'th', 'e.', '\" ', '\"m', 'an', 'y ', 'th', 'an', 'ks', ', ', 'pi', 'f-', 'pa', 'f-', 'po', 'lt', 'ri', 'e.', '\" ', '\"m', 'ay', ' i', '\\nb', 'e ', 'al', 'lo', 'we', 'd ', 'to', ' h', 'av', 'e ', 'yo', 'ur', ' d', 'au', 'gh', 'te']\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "  \n",
    "  with open(filename) as f:\n",
    "    data = tf.compat.as_str(f.read())\n",
    "    # make all the text lowercase\n",
    "    data = data.lower()\n",
    "    data = list(data)\n",
    "  return data\n",
    "\n",
    "documents = []\n",
    "global documents\n",
    "for i in range(num_files):    \n",
    "    print('\\nProcessing file %s'%os.path.join(dir_name,filenames[i]))\n",
    "    chars = read_data(os.path.join(dir_name,filenames[i]))\n",
    "    \n",
    "    # Breaking the text into bigrams\n",
    "    two_grams = [''.join(chars[ch_i:ch_i+2]) for ch_i in range(0,len(chars)-2,2)]\n",
    "    # Creates a list of lists with the bigrams (outer loop different stories)\n",
    "    documents.append(two_grams)\n",
    "    print('Data size (Characters) (Document %d) %d' %(i,len(two_grams)))\n",
    "    print('Sample string (Document %d) %s'%(i,two_grams[:50]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the Dictionaries (Bigrams)\n",
    "Builds the following. To understand each of these elements, let us also assume the text \"I like to go to school\"\n",
    "\n",
    "* `dictionary`: maps a string word to an ID (e.g. {I:0, like:1, to:2, go:3, school:4})\n",
    "* `reverse_dictionary`: maps an ID to a string word (e.g. {0:I, 1:like, 2:to, 3:go, 4:school}\n",
    "* `count`: List of list of (word, frequency) elements (e.g. [(I,1),(like,1),(to,2),(go,1),(school,1)]\n",
    "* `data` : Contain the string of text we read, where string words are replaced with word IDs (e.g. [0, 1, 2, 3, 2, 4])\n",
    "\n",
    "It also introduces an additional special token `UNK` to denote rare words to are too rare to make use of."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "449177 Characters found.\n",
      "Most common words (+UNK) [('e ', 15229), ('he', 15164), (' t', 13443), ('th', 13076), ('d ', 10687)]\n",
      "Least common words (+UNK) [('rz', 1), ('zi', 1), ('i?', 1), ('\\ts', 1), ('\".', 1), ('hc', 1), ('sd', 1), ('z ', 1), ('m?', 1), ('\\tc', 1), ('oz', 1), ('iq', 1), ('pw', 1), ('tz', 1), ('yr', 1)]\n",
      "Sample data [15, 28, 86, 23, 3, 95, 74, 11, 2, 16]\n",
      "Sample data [22, 156, 25, 37, 82, 185, 43, 9, 90, 19]\n",
      "Vocabulary:  544\n"
     ]
    }
   ],
   "source": [
    "def build_dataset(documents):\n",
    "    chars = []\n",
    "    # This is going to be a list of lists\n",
    "    # Where the outer list denote each document\n",
    "    # and the inner lists denote words in a given document\n",
    "    data_list = []\n",
    "  \n",
    "    for d in documents:\n",
    "        chars.extend(d)\n",
    "    print('%d Characters found.'%len(chars))\n",
    "    count = []\n",
    "    # Get the bigram sorted by their frequency (Highest comes first)\n",
    "    count.extend(collections.Counter(chars).most_common())\n",
    "    \n",
    "    # Create an ID for each bigram by giving the current length of the dictionary\n",
    "    # And adding that item to the dictionary\n",
    "    # Start with 'UNK' that is assigned to too rare words\n",
    "    dictionary = dict({'UNK':0})\n",
    "    for char, c in count:\n",
    "        # Only add a bigram to dictionary if its frequency is more than 10\n",
    "        if c > 10:\n",
    "            dictionary[char] = len(dictionary)    \n",
    "    \n",
    "    unk_count = 0\n",
    "    # Traverse through all the text we have\n",
    "    # to replace each string word with the ID of the word\n",
    "    for d in documents:\n",
    "        data = list()\n",
    "        for char in d:\n",
    "            # If word is in the dictionary use the word ID,\n",
    "            # else use the ID of the special token \"UNK\"\n",
    "            if char in dictionary:\n",
    "                index = dictionary[char]        \n",
    "            else:\n",
    "                index = dictionary['UNK']\n",
    "                unk_count += 1\n",
    "            data.append(index)\n",
    "            \n",
    "        data_list.append(data)\n",
    "        \n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) \n",
    "    return data_list, count, dictionary, reverse_dictionary\n",
    "\n",
    "global data_list, count, dictionary, reverse_dictionary,vocabulary_size\n",
    "\n",
    "# Print some statistics about data\n",
    "data_list, count, dictionary, reverse_dictionary = build_dataset(documents)\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Least common words (+UNK)', count[-15:])\n",
    "print('Sample data', data_list[0][:10])\n",
    "print('Sample data', data_list[1][:10])\n",
    "print('Vocabulary: ',len(dictionary))\n",
    "vocabulary_size = len(dictionary)\n",
    "del documents  # To reduce memory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Batches of Data\n",
    "The following object generates a batch of data which will be used to train the RNN. More specifically the generator breaks a given sequence of words into `batch_size` segments. We also maintain a cursor for each segment. So whenever we create a batch of data, we sample one item from each segment and update the cursor of each segment. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Unrolled index 0\n",
      "\tInputs:\n",
      "\te  (1), \tki (131), \t d (48), \t w (11), \tbe (70), \n",
      "\tOutput:\n",
      "\tli (98), \tng (33), \tau (195), \ter (14), \tau (195), \n",
      "\n",
      "Unrolled index 1\n",
      "\tInputs:\n",
      "\tli (98), \tng (33), \tau (195), \ter (14), \tau (195), \n",
      "\tOutput:\n",
      "\tve (41), \t\n",
      "w (169), \tgh (106), \te  (1), \tti (112), \n",
      "\n",
      "Unrolled index 2\n",
      "\tInputs:\n",
      "\tve (41), \t\n",
      "w (169), \tgh (106), \te  (1), \tti (112), \n",
      "\tOutput:\n",
      "\td  (5), \tho (62), \tte (61), \tal (84), \tfu (229), \n",
      "\n",
      "Unrolled index 3\n",
      "\tInputs:\n",
      "\td  (5), \tho (62), \tte (61), \tal (84), \tfu (229), \n",
      "\tOutput:\n",
      "\ta  (82), \tse (58), \trs (137), \tl  (57), \tl, (257), \n",
      "\n",
      "Unrolled index 4\n",
      "\tInputs:\n",
      "\ta  (82), \tse (58), \trs (137), \tl  (57), \tbe (70), \n",
      "\tOutput:\n",
      "\tki (131), \t d (48), \t w (11), \tbe (70), \tau (195), "
     ]
    }
   ],
   "source": [
    "class DataGeneratorOHE(object):\n",
    "    \n",
    "    def __init__(self,text,batch_size,num_unroll):\n",
    "        # Text where a bigram is denoted by its ID\n",
    "        self._text = text\n",
    "        # Number of bigrams in the text\n",
    "        self._text_size = len(self._text)\n",
    "        # Number of datapoints in a batch of data\n",
    "        self._batch_size = batch_size\n",
    "        # Num unroll is the number of steps we unroll the RNN in a single training step\n",
    "        # This relates to the truncated backpropagation we discuss in Chapter 6 text\n",
    "        self._num_unroll = num_unroll\n",
    "        # We break the text in to several segments and the batch of data is sampled by\n",
    "        # sampling a single item from a single segment\n",
    "        self._segments = self._text_size//self._batch_size\n",
    "        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n",
    "        \n",
    "    def next_batch(self):\n",
    "        '''\n",
    "        Generates a single batch of data\n",
    "        '''\n",
    "        # Train inputs (one-hot-encoded) and train outputs (one-hot-encoded)\n",
    "        batch_data = np.zeros((self._batch_size,vocabulary_size),dtype=np.float32)\n",
    "        batch_labels = np.zeros((self._batch_size,vocabulary_size),dtype=np.float32)\n",
    "        \n",
    "        # Fill in the batch datapoint by datapoint\n",
    "        for b in range(self._batch_size):\n",
    "            # If the cursor of a given segment exceeds the segment length\n",
    "            # we reset the cursor back to the beginning of that segment\n",
    "            if self._cursor[b]+1>=self._text_size:\n",
    "                self._cursor[b] = b * self._segments\n",
    "            \n",
    "            # Add the text at the cursor as the input\n",
    "            batch_data[b,self._text[self._cursor[b]]] = 1.0\n",
    "            # Add the preceding bigram as the label to be predicted\n",
    "            batch_labels[b,self._text[self._cursor[b]+1]]= 1.0                       \n",
    "            # Update the cursor\n",
    "            self._cursor[b] = (self._cursor[b]+1)%self._text_size\n",
    "                    \n",
    "        return batch_data,batch_labels\n",
    "        \n",
    "    def unroll_batches(self):\n",
    "        '''\n",
    "        This produces a list of num_unroll batches\n",
    "        as required by a single step of training of the RNN\n",
    "        '''\n",
    "        unroll_data,unroll_labels = [],[]\n",
    "        for ui in range(self._num_unroll):\n",
    "            data, labels = self.next_batch()            \n",
    "            unroll_data.append(data)\n",
    "            unroll_labels.append(labels)\n",
    "        \n",
    "        return unroll_data, unroll_labels\n",
    "    \n",
    "    def reset_indices(self):\n",
    "        '''\n",
    "        Used to reset all the cursors if needed\n",
    "        '''\n",
    "        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n",
    "        \n",
    "# Running a tiny set to see if things are correct\n",
    "dg = DataGeneratorOHE(data_list[0][25:50],5,5)\n",
    "u_data, u_labels = dg.unroll_batches()\n",
    "\n",
    "# Iterate through each data batch in the unrolled set of batches\n",
    "for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):   \n",
    "    print('\\n\\nUnrolled index %d'%ui)\n",
    "    dat_ind = np.argmax(dat,axis=1)\n",
    "    lbl_ind = np.argmax(lbl,axis=1)\n",
    "    print('\\tInputs:')\n",
    "    for sing_dat in dat_ind:\n",
    "        print('\\t%s (%d)'%(reverse_dictionary[sing_dat],sing_dat),end=\", \")\n",
    "    print('\\n\\tOutput:')\n",
    "    for sing_lbl in lbl_ind:        \n",
    "        print('\\t%s (%d)'%(reverse_dictionary[sing_lbl],sing_lbl),end=\", \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the LSTM, LSTM with Peepholes and GRUs\n",
    "\n",
    "* A LSTM has 5 main components\n",
    "  * Cell state, Hidden state, Input gate, Forget gate, Output gate\n",
    "* A LSTM with peephole connections\n",
    "  * Introduces several new sets of weights that connects the cell state to the gates\n",
    "* A GRU has 3 main components\n",
    "  * Hidden state, Reset gate and a Update gate\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining hyperparameters\n",
    "\n",
    "Here we define several hyperparameters and are very similar to the ones we defined in Chapter 6. However additionally we use dropout; a technique that helps to avoid overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_nodes = 128\n",
    "batch_size = 64\n",
    "num_unrollings = 50\n",
    "dropout = 0.2\n",
    "\n",
    "# Use this in the CSV filename when saving\n",
    "# when using dropout\n",
    "filename_extension = ''\n",
    "if dropout>0.0:\n",
    "    filename_extension = '_dropout'\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Inputs and Outputs\n",
    "\n",
    "In the code we define two different types of inputs. \n",
    "* Training inputs (The stories we downloaded) (batch_size > 1 with unrolling)\n",
    "* Validation inputs (An unseen validation dataset) (bach_size =1, no unrolling)\n",
    "* Test input (New story we are going to generate) (batch_size=1, no unrolling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Training Input data.\n",
    "train_inputs, train_labels = [],[]\n",
    "\n",
    "# Defining unrolled training inputs\n",
    "for ui in range(num_unrollings):\n",
    "    train_inputs.append(tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size],name='train_inputs_%d'%ui))\n",
    "    train_labels.append(tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size], name = 'train_labels_%d'%ui))\n",
    "\n",
    "valid_inputs = tf.placeholder(tf.float32, shape=[1, vocabulary_size])\n",
    "valid_labels = tf.placeholder(tf.float32, shape=[1, vocabulary_size])\n",
    "\n",
    "# Text generation: batch 1, no unrolling.\n",
    "test_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Model Parameters and Cell Computation\n",
    "\n",
    "We define parameters and cell computation functions for all the different variants (LSTM, LSTM with peepholes and GRUs). **Make sure you only run a single cell withing this section (either the LSTM/ LSTM with peepholes or GRUs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standard LSTM\n",
    "\n",
    "Here we define the parameters and the cell computation function for a standard LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Input gate (i_t) - How much memory to write to cell state\n",
    "# Connects the current input to the input gate\n",
    "ix = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the input gate\n",
    "im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the input gate\n",
    "ib = tf.Variable(tf.random_uniform([1, num_nodes],-0.02, 0.02))\n",
    "\n",
    "# Forget gate (f_t) - How much memory to discard from cell state\n",
    "# Connects the current input to the forget gate\n",
    "fx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the forget gate\n",
    "fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the forget gate\n",
    "fb = tf.Variable(tf.random_uniform([1, num_nodes],-0.02, 0.02))\n",
    "\n",
    "# Candidate value (c~_t) - Used to compute the current cell state\n",
    "# Connects the current input to the candidate\n",
    "cx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the candidate\n",
    "cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the candidate\n",
    "cb = tf.Variable(tf.random_uniform([1, num_nodes],-0.02,0.02))\n",
    "\n",
    "# Output gate - How much memory to output from the cell state\n",
    "# Connects the current input to the output gate\n",
    "ox = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the output gate\n",
    "om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the output gate\n",
    "ob = tf.Variable(tf.random_uniform([1, num_nodes],-0.02,0.02))\n",
    "\n",
    "\n",
    "# Softmax Classifier weights and biases.\n",
    "w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], stddev=0.02))\n",
    "b = tf.Variable(tf.random_uniform([vocabulary_size],-0.02,0.02))\n",
    "\n",
    "# Variables saving state across unrollings.\n",
    "# Hidden state\n",
    "saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "# Cell state\n",
    "saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "\n",
    "saved_valid_output = tf.Variable(tf.zeros([1, num_nodes]), trainable=False)\n",
    "saved_valid_state = tf.Variable(tf.zeros([1, num_nodes]), trainable=False)\n",
    "\n",
    "# Same variables for testing phase\n",
    "saved_test_output = tf.Variable(tf.zeros([1, num_nodes]),trainable=False)\n",
    "saved_test_state = tf.Variable(tf.zeros([1, num_nodes]),trainable=False)\n",
    "\n",
    "algorithm = 'lstm'\n",
    "filename_to_save = algorithm + filename_extension +'.csv'\n",
    "# Definition of the cell computation.\n",
    "def lstm_cell(i, o, state):\n",
    "    \"\"\"Create an LSTM cell\"\"\"\n",
    "    input_gate = tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)\n",
    "    forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)\n",
    "    update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n",
    "    state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "    output_gate = tf.sigmoid(tf.matmul(i, ox) + tf.matmul(o, om) + ob)\n",
    "    return output_gate * tf.tanh(state), state\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### LSTMs with Peephole Connections\n",
    "\n",
    "We define the parameters and cell computation for a LSTM with peepholes. Note that we are using diagonal peephole connections (for more details refer the text)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters:\n",
    "# Input gate: input, previous output, and bias.\n",
    "ix = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.01))\n",
    "im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.01))\n",
    "ic = tf.Variable(tf.truncated_normal([1,num_nodes], stddev=0.01))\n",
    "ib = tf.Variable(tf.random_uniform([1, num_nodes],0.0, 0.01))\n",
    "# Forget gate: input, previous output, and bias.\n",
    "fx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.01))\n",
    "fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.01))\n",
    "fc = tf.Variable(tf.truncated_normal([1,num_nodes], stddev=0.01))\n",
    "fb = tf.Variable(tf.random_uniform([1, num_nodes],0.0, 0.01))\n",
    "# Memory cell: input, state and bias.                             \n",
    "cx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.01))\n",
    "cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.01))\n",
    "cb = tf.Variable(tf.random_uniform([1, num_nodes],0.0,0.01))\n",
    "# Output gate: input, previous output, and bias.\n",
    "ox = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.01))\n",
    "om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.01))\n",
    "oc = tf.Variable(tf.truncated_normal([1,num_nodes], stddev=0.01))\n",
    "ob = tf.Variable(tf.random_uniform([1, num_nodes],0.0,0.01))\n",
    "\n",
    "# Softmax Classifier weights and biases.\n",
    "w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], stddev=0.01))\n",
    "b = tf.Variable(tf.random_uniform([vocabulary_size],0.0,0.01))\n",
    "\n",
    "# Variables saving state across unrollings.\n",
    "saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "\n",
    "saved_valid_output = tf.Variable(tf.zeros([1, num_nodes]), trainable=False)\n",
    "saved_valid_state = tf.Variable(tf.zeros([1, num_nodes]), trainable=False)\n",
    "\n",
    "saved_test_output = tf.Variable(tf.zeros([1, num_nodes]), trainable=False)\n",
    "saved_test_state = tf.Variable(tf.zeros([1, num_nodes]), trainable=False)\n",
    "\n",
    "algorithm = 'lstm_peephole'\n",
    "filename_to_save = algorithm + filename_extension +'.csv'\n",
    "# Definition of the cell computation.\n",
    "def lstm_with_peephole_cell(i, o, state):\n",
    "    '''\n",
    "    LSTM with peephole connections\n",
    "    Our implementation for peepholes is based on \n",
    "    https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43905.pdf    \n",
    "    '''\n",
    "    input_gate = tf.sigmoid(tf.matmul(i, ix) + state*ic + tf.matmul(o, im) + ib)\n",
    "    forget_gate = tf.sigmoid(tf.matmul(i, fx) + state*fc + tf.matmul(o, fm) + fb)\n",
    "    update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n",
    "    state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "    output_gate = tf.sigmoid(tf.matmul(i, ox) + state*oc + tf.matmul(o, om) + ob)\n",
    "\n",
    "    return output_gate * tf.tanh(state), state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gated Recurrent Units (GRUs)\n",
    "\n",
    "Finally we define the parameters and cell computations for the GRU cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters:\n",
    "# Reset gate: input, previous output, and bias.\n",
    "rx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.01))\n",
    "rh = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.01))\n",
    "rb = tf.Variable(tf.random_uniform([1, num_nodes],0.0, 0.01))\n",
    "\n",
    "# Hidden State: input, previous output, and bias.\n",
    "hx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.01))\n",
    "hh = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.01))\n",
    "hb = tf.Variable(tf.random_uniform([1, num_nodes],0.0, 0.01))\n",
    "\n",
    "# Update gate: input, previous output, and bias.\n",
    "zx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.01))\n",
    "zh = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.01))\n",
    "zb = tf.Variable(tf.random_uniform([1, num_nodes],0.0, 0.01))\n",
    "\n",
    "# Softmax Classifier weights and biases.\n",
    "w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], stddev=0.01))\n",
    "b = tf.Variable(tf.random_uniform([vocabulary_size],0.0,0.01))\n",
    "\n",
    "# Variables saving state across unrollings.\n",
    "saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "saved_valid_output = tf.Variable(tf.zeros([1, num_nodes]),trainable=False)\n",
    "saved_test_output = tf.Variable(tf.zeros([1, num_nodes]),trainable=False)\n",
    "\n",
    "algorithm = 'gru'\n",
    "filename_to_save = algorithm + filename_extension +'.csv'\n",
    "\n",
    "# Definition of the cell computation.\n",
    "def gru_cell(i, o):\n",
    "    \"\"\"Create a GRU cell.\"\"\"\n",
    "    reset_gate = tf.sigmoid(tf.matmul(i, rx) + tf.matmul(o, rh) + rb)\n",
    "    h_tilde = tf.tanh(tf.matmul(i,hx) + tf.matmul(reset_gate * o, hh) + hb)\n",
    "    z = tf.sigmoid(tf.matmul(i,zx) + tf.matmul(o, zh) + zb)\n",
    "    h = (1-z)*o + z*h_tilde\n",
    "    \n",
    "    return h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining LSTM/GRU/LSTM-Peephole Computations\n",
    "Here first we define the LSTM cell computations as a consice function. Then we use this function to define training and test-time inference logic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =========================================================\n",
    "#Training related inference logic\n",
    "\n",
    "# Keeps the calculated state outputs in all the unrollings\n",
    "# Used to calculate loss\n",
    "outputs = list()\n",
    "\n",
    "# These two python variables are iteratively updated\n",
    "# at each step of unrolling\n",
    "output = saved_output\n",
    "if algorithm=='lstm' or algorithm=='lstm_peephole':\n",
    "  state = saved_state\n",
    "\n",
    "# Compute the hidden state (output) and cell state (state)\n",
    "# recursively for all the steps in unrolling\n",
    "# Note: there is no cell state for GRUs\n",
    "for i in train_inputs:\n",
    "    if algorithm=='lstm':\n",
    "      output, state = lstm_cell(i, output, state)\n",
    "      train_state_update_ops = [saved_output.assign(output),\n",
    "                                saved_state.assign(state)]\n",
    "    elif algorithm=='lstm_peephole':\n",
    "      output, state = lstm_with_peephole_cell(i, output, state)\n",
    "      train_state_update_ops = [saved_output.assign(output),\n",
    "                                saved_state.assign(state)]\n",
    "    elif algorithm=='gru':\n",
    "      output = gru_cell(i, output)\n",
    "      train_state_update_ops = [saved_output.assign(output)]\n",
    "        \n",
    "    output = tf.nn.dropout(output,keep_prob=1.0-dropout)\n",
    "    # Append each computed output value\n",
    "    outputs.append(output)\n",
    "\n",
    "# calculate the score values\n",
    "logits = tf.matmul(tf.concat(axis=0, values=outputs), w) + b\n",
    "    \n",
    "# Compute predictions.\n",
    "train_prediction = tf.nn.softmax(logits)\n",
    "\n",
    "# Compute training perplexity\n",
    "train_perplexity_without_exp = tf.reduce_sum(tf.concat(train_labels,0)*-tf.log(tf.concat(train_prediction,0)+1e-10))/(num_unrollings*batch_size)\n",
    "\n",
    "# ========================================================================\n",
    "# Validation phase related inference logic\n",
    "\n",
    "valid_output = saved_valid_output\n",
    "if algorithm=='lstm' or algorithm=='lstm_peephole':\n",
    "  valid_state = saved_valid_state\n",
    "\n",
    "# Compute the LSTM cell output for validation data\n",
    "if algorithm=='lstm':\n",
    "    valid_output, valid_state = lstm_cell(\n",
    "        valid_inputs, saved_valid_output, saved_valid_state)\n",
    "    valid_state_update_ops = [saved_valid_output.assign(valid_output),\n",
    "                                saved_valid_state.assign(valid_state)]\n",
    "    \n",
    "elif algorithm=='lstm_peephole':\n",
    "    valid_output, valid_state = lstm_with_peephole_cell(\n",
    "        valid_inputs, saved_valid_output, saved_valid_state)\n",
    "    valid_state_update_ops = [saved_valid_output.assign(valid_output),\n",
    "                                saved_valid_state.assign(valid_state)]\n",
    "elif algorithm=='gru':\n",
    "    valid_output = gru_cell(valid_inputs, valid_output)\n",
    "    valid_state_update_ops = [saved_valid_output.assign(valid_output)]\n",
    "\n",
    "valid_logits = tf.nn.xw_plus_b(valid_output, w, b)\n",
    "# Make sure that the state variables are updated\n",
    "# before moving on to the next iteration of generation\n",
    "with tf.control_dependencies(valid_state_update_ops):\n",
    "    valid_prediction = tf.nn.softmax(valid_logits)\n",
    "\n",
    "# Compute validation perplexity\n",
    "valid_perplexity_without_exp = tf.reduce_sum(valid_labels*-tf.log(valid_prediction+1e-10))\n",
    "\n",
    "# ========================================================================\n",
    "# Testing phase related inference logic\n",
    "\n",
    "# Compute the LSTM cell output for testing data\n",
    "if algorithm=='lstm':\n",
    "  test_output, test_state = lstm_cell(test_input, saved_test_output, saved_test_state)\n",
    "  test_state_update_ops = [saved_test_output.assign(test_output),\n",
    "                            saved_test_state.assign(test_state)]\n",
    "elif algorithm=='lstm_peephole':\n",
    "  test_output, test_state = lstm_with_peephole_cell(test_input, saved_test_output, saved_test_state)\n",
    "  test_state_update_ops = [saved_test_output.assign(test_output),\n",
    "                            saved_test_state.assign(test_state)]\n",
    "elif algorithm=='gru':\n",
    "  test_output = gru_cell(test_input, saved_test_output)\n",
    "  test_state_update_ops = [saved_test_output.assign(test_output)]\n",
    "\n",
    "# Make sure that the state variables are updated\n",
    "# before moving on to the next iteration of generation\n",
    "with tf.control_dependencies(test_state_update_ops):\n",
    "    test_prediction = tf.nn.softmax(tf.nn.xw_plus_b(test_output, w, b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating LSTM Loss\n",
    "We calculate the training loss of the LSTM here. It's a typical cross entropy loss calculated over all the scores we obtained for training data (`loss`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Before calcualting the training loss,\n",
    "# save the hidden state and the cell state to\n",
    "# their respective TensorFlow variables\n",
    "with tf.control_dependencies(train_state_update_ops):\n",
    "\n",
    "    # Calculate the training loss by\n",
    "    # concatenating the results from all the unrolled time steps\n",
    "    loss = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits_v2(\n",
    "        logits=logits, labels=tf.concat(axis=0, values=train_labels)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resetting Operations for Resetting Hidden States\n",
    "Sometimes the state variable needs to be reset (e.g. when starting predictions at a beginning of a new epoch). But since GRU doesn't have a cell state we have a conditioned reset_state ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-105-387b34694363>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m     \u001b[1;31m# Reset test state. We use imputations in the test state reset\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m     \u001b[0mreset_test_state\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0msaved_test_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0massign\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandom_normal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_nodes\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mstddev\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.01\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\ops\\random_ops.py\u001b[0m in \u001b[0;36mrandom_normal\u001b[1;34m(shape, mean, stddev, dtype, seed, name)\u001b[0m\n\u001b[0;32m     76\u001b[0m     \u001b[0mseed1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseed2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrandom_seed\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_seed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     77\u001b[0m     rnd = gen_random_ops._random_standard_normal(\n\u001b[1;32m---> 78\u001b[1;33m         shape_tensor, dtype, seed=seed1, seed2=seed2)\n\u001b[0m\u001b[0;32m     79\u001b[0m     \u001b[0mmul\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrnd\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mstddev_tensor\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     80\u001b[0m     \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmul\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmean_tensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\ops\\gen_random_ops.py\u001b[0m in \u001b[0;36m_random_standard_normal\u001b[1;34m(shape, dtype, seed, seed2, name)\u001b[0m\n\u001b[0;32m    417\u001b[0m     _, _, _op = _op_def_lib._apply_op_helper(\n\u001b[0;32m    418\u001b[0m         \u001b[1;34m\"RandomStandardNormal\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mseed\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 419\u001b[1;33m         seed2=seed2, name=name)\n\u001b[0m\u001b[0;32m    420\u001b[0m     \u001b[0m_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_op\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    421\u001b[0m     \u001b[0m_inputs_flat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_op\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\u001b[0m in \u001b[0;36m_apply_op_helper\u001b[1;34m(self, op_type_name, name, **keywords)\u001b[0m\n\u001b[0;32m    360\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    361\u001b[0m     \u001b[1;31m# Check for deprecation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 362\u001b[1;33m     \u001b[0mdeprecation_version\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mop_def\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdeprecation\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mversion\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    363\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mdeprecation_version\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    364\u001b[0m       \u001b[0mproducer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph_def_versions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproducer\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "if algorithm=='lstm' or algorithm=='lstm_peephole':\n",
    "    # Reset train state\n",
    "    reset_train_state = tf.group(tf.assign(saved_state, tf.zeros([batch_size, num_nodes])),\n",
    "                          tf.assign(saved_output, tf.zeros([batch_size, num_nodes])))\n",
    "\n",
    "    reset_valid_state = tf.group(tf.assign(saved_valid_state, tf.zeros([1, num_nodes])),\n",
    "                          tf.assign(saved_valid_output, tf.zeros([1, num_nodes])))\n",
    "    \n",
    "    # Reset test state. We use imputations in the test state reset\n",
    "    reset_test_state = tf.group(\n",
    "        saved_test_output.assign(tf.random_normal([1, num_nodes],stddev=0.01)),\n",
    "        saved_test_state.assign(tf.random_normal([1, num_nodes],stddev=0.01)))\n",
    "    \n",
    "elif algorithm=='gru':\n",
    "    # Reset train state\n",
    "    reset_train_state = [tf.assign(saved_output, tf.zeros([batch_size, num_nodes]))]\n",
    "\n",
    "    # Reset valid state\n",
    "    reset_valid_state = [tf.assign(saved_valid_output, tf.zeros([1, num_nodes]))]\n",
    "    \n",
    "    # Reset test state. We use imputations in the test state reset\n",
    "    reset_test_state = [saved_test_output.assign(tf.random_normal([1, num_nodes],stddev=0.01))]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Learning Rate and the Optimizer with Gradient Clipping\n",
    "Here we define the learning rate and the optimizer we're going to use. We will be using the Adam optimizer as it is one of the best optimizers out there. Furthermore we use gradient clipping to prevent any gradient explosions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Used for decaying learning rate\n",
    "gstep = tf.Variable(0, trainable=False)\n",
    "\n",
    "# Running this operation will cause the value of gstep\n",
    "# to increase, while in turn reducing the learning rate\n",
    "inc_gstep = tf.assign(gstep, gstep+1)\n",
    "\n",
    "# Decays learning rate everytime the gstep increases\n",
    "tf_learning_rate = tf.train.exponential_decay(0.001,gstep,decay_steps=1, decay_rate=0.5)\n",
    "\n",
    "# Adam Optimizer. And gradient clipping.\n",
    "optimizer = tf.train.AdamOptimizer(tf_learning_rate)\n",
    "\n",
    "gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "# Clipping gradients\n",
    "gradients, _ = tf.clip_by_global_norm(gradients, 5.0)\n",
    "\n",
    "optimizer = optimizer.apply_gradients(\n",
    "    zip(gradients, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Greedy Sampling to Break the Repetition\n",
    "Here we write some simple logic to break the repetition in text. Specifically instead of always getting the word that gave this highest prediction probability, we sample randomly where the probability of being selected given by their prediction probabilities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sample(distribution):\n",
    "  '''Greedy Sampling\n",
    "  We pick the three best predictions given by the LSTM and sample\n",
    "  one of them with very high probability of picking the best one'''\n",
    "  best_inds = np.argsort(distribution)[-3:]\n",
    "  best_probs = distribution[best_inds]/np.sum(distribution[best_inds])\n",
    "  best_idx = np.random.choice(best_inds,p=best_probs)\n",
    "  return best_idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running the LSTM to Generate Text\n",
    "\n",
    "Here we train the model on the available data and generate text using the trained model for several steps. From each document we extract text for `steps_per_document` steps to train the model on. We also report the train perplexity at the end of each step. Finally we test the model by asking it to generate some new text starting from a randomly picked bigram."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning rate Decay Logic\n",
    "\n",
    "Here we define the logic to decrease learning rate whenever the validation perplexity does not decrease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning rate decay related\n",
    "# If valid perpelxity does not decrease\n",
    "# continuously for this many epochs\n",
    "# decrease the learning rate\n",
    "decay_threshold = 5\n",
    "# Keep counting perplexity increases\n",
    "decay_count = 0\n",
    "\n",
    "min_perplexity = 1e10\n",
    "\n",
    "# Learning rate decay logic\n",
    "def decay_learning_rate(session, v_perplexity):\n",
    "  global decay_threshold, decay_count, min_perplexity  \n",
    "  # Decay learning rate\n",
    "  if v_perplexity < min_perplexity:\n",
    "    decay_count = 0\n",
    "    min_perplexity= v_perplexity\n",
    "  else:\n",
    "    decay_count += 1\n",
    "\n",
    "  if decay_count >= decay_threshold:\n",
    "    print('\\t Reducing learning rate')\n",
    "    decay_count = 0\n",
    "    session.run(inc_gstep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running Training, Validation and Generation\n",
    "\n",
    "We traing the LSTM on existing training data, check the validaiton perplexity on an unseen chunk of text and generate a fresh segment of text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized Global Variables \n",
      "(98).(25).(91).(5).(88).(49).(85).(96).(14).(73).\n",
      "Average loss at step 1: 4.500272\n",
      "\tPerplexity at step 1: 90.041577\n",
      "\n",
      "Valid Perplexity: 53.93\n",
      "\n",
      "Generated Text after epoch 0 ... \n",
      "======================== New text Segment ==========================\n",
      "\t her, it the spirit, \"one his that to and said the money the spirit, and\n",
      "here, and have, and\n",
      "the gold all wile you that it the morester, and the spirit and had with ith the spirit, and hered hen hen that have the spirit, and the spiras, i will said the spirout.  i will said, \"i will wout on that to and said, \"i wither in the bover, \"the spirit, \"one the father, \"i will said, \"the boy, and had to that it to the have\n",
      "to the father, \"and\n",
      "here, that had come you came, and here, and here, and, \"the spirour wither as the money the spirler the spirit, i must the bected hen the boy that to you with the father to the first had come to the fore the monen the spneit, and have, answered and said, \"the gon that and he could hey the money you will sood that in ther, what have the spirit.\"  the gold for to you the more his plaster, \"i will the fathen all had come, and wound in the boke, i will had come to the father, it you that then your had then your as you came, and have, and hen the boner to the had\n",
      "====================================================================\n",
      "\n",
      "(49).(87).(32).(14).(4).(51).(90).(16).(60).(43).\n",
      "Average loss at step 2: 2.719010\n",
      "\tPerplexity at step 2: 15.165307\n",
      "\n",
      "Valid Perplexity: 38.30\n",
      "\n",
      "Generated Text after epoch 1 ... \n",
      "======================== New text Segment ==========================\n",
      "\t r, but the name, and the queen's allered that is name is the name, the queen was the name, and the queen was hease, and the little hands that he more himself in, and the man came the manikin two man, and the manikin was jumping, he pulled at his left leg so hard that is name in, and the little man whow to her leg were in his the deall the manikin his whole leg the queen's dever had told the names your name, the my the names that he plunged his right the little man came in, and the manikin she knew, that to\n",
      "the\n",
      "name.  but the name in the queen, what is my name in his whole his told you thatUNK\n",
      "the devil has told yound the manikin said, is two my name.\n",
      "\n",
      "on the little man came in, and foot so\n",
      "the\n",
      "manikin said, not the manikin said, is that is not not no the little man, and all the little man cantle the\n",
      "nauntribs, of the little man, and then in the little hands and the queen's dever's child, what is you thatUNK\n",
      "the dever has hold.  but\n",
      "she had in, and\n",
      "the little man, and in the night, that int\n",
      "====================================================================\n",
      "\n",
      "(48).(25).(81).(71).(45).(13).(0).(53).(28).(40).\n",
      "Average loss at step 3: 2.477577\n",
      "\tPerplexity at step 3: 11.912361\n",
      "\n",
      "Valid Perplexity: 32.62\n",
      "\n",
      "Generated Text after epoch 2 ... \n",
      "======================== New text Segment ==========================\n",
      "\t asked his which she put of in two egg-should now will been the must splet down and said, i have done.  when he said, i am to humble you can been to the king's heart, and she had driven her the most splendown the king throuhbeard of the king's daughter was too.  i wish you will\n",
      "happened to the corner of the king's evil danced, and that the most splet son her\n",
      "for and the king's door this did now began in that her and will\n",
      "be on which your promised that it down on this will down on the maid, i with you had to the\n",
      "cornest.  i have been to the heart, and she was laughter and dide once with they down the maid to the comforted the pon the kindly, there\n",
      "too.  and her and\n",
      "then the prode, who she said to this days that when the maidUNKin-waiting came and put on her to the maidUNKin-waiting came and put on her the most splend and were of the hand that with your father and that the poor and\n",
      "been she was to this days wedding.\n",
      "then the door, which your wife.  but he said, be court sprangs the kind's ear\n",
      "====================================================================\n",
      "\n",
      "(78).(49).(12).(40).(27).(34).(89).(28).(66).(58).\n",
      "Average loss at step 4: 2.076020\n",
      "\tPerplexity at step 4: 7.972671\n",
      "\n",
      "Valid Perplexity: 50.50\n",
      "\n",
      "Generated Text after epoch 3 ... \n",
      "======================== New text Segment ==========================\n",
      "\t out of which to the ground and broke.  then they bought him his eyes for a while, and presently\n",
      "began to gather to the table, and\n",
      "henceforth always let him eat with them, and likewise said nothing if\n",
      "he did spill a little of anything.  and they took the old grandfather to the table, and\n",
      "henceforth always let him eat with them, and likewise said nothing if\n",
      "he did spill a little of anything.  i am making a little trough,\n",
      "answered the child, for father and mother to eat with them, and likewise said nothing if\n",
      "he did spill a little of anything.  the old grandfather to the table while, and presently\n",
      "began to cry.  then they took the old grandfather to the table, and\n",
      "henceforth always let him eat with them, for a while, and presently\n",
      "began to cry.  then they took the old grandfather to gather to the table, and\n",
      "henceforth always let him eat with them, and likewise said nothing if\n",
      "he did nothing if\n",
      "he did spill a little of anything.  i am making a little trough,\n",
      "answered the child, for for a whi\n",
      "====================================================================\n",
      "\n",
      "(72).(5).(55).(2).(42).(75).(57).(80).(47).(14).\n",
      "Average loss at step 5: 2.553451\n",
      "\tPerplexity at step 5: 12.851376\n",
      "\n",
      "Valid Perplexity: 23.96\n",
      "\n",
      "Generated Text after epoch 4 ... \n",
      "======================== New text Segment ==========================\n",
      "\t \n",
      "wither, as which,\n",
      "so the king, which should not he was came to the little said me that he tailor, and after her\n",
      "father and bear, the little tailor, when he who had been\n",
      "boil, and they were\n",
      "one of the king, and they were to be dought the little tailor was comforted the tree.  then the king, who hans went one boy, and after her\n",
      "father's death became to his have not the will tailor again the two\n",
      "other so low.  i smote not\n",
      "liked the little tailor, who was at one of them to the board against the tailor.  when the wild boy, and after her\n",
      "father, and then the little tailor was and remadeed, and the little tailor and the little tailor, and tailor half of the tailor standing, and it was thouse, what he had heard the tailor so the little said, the tailor had faller asleep the treat with them, and then it, and they who who was no one of the king was thoubly and they will for\n",
      "him, and then the king, who was caught, but they will forest again, who had heard the tree.  the two giants and said than he\n",
      "====================================================================\n",
      "\n",
      "(25).(89).(52).(2).(63).(74).(61).(10).(56).(64).\n",
      "Average loss at step 6: 2.086129\n",
      "\tPerplexity at step 6: 8.053676\n",
      "\n",
      "Valid Perplexity: 24.38\n",
      "\n",
      "Generated Text after epoch 5 ... \n",
      "======================== New text Segment ==========================\n",
      "\t e the bridegroom with the nut.  immediately\n",
      "came and said, nevery that they were she came to the bride she was ablew on the bride was on their deathe great before in the midst.  and the bridegroom with the griffin the griffind there, but the bride was\n",
      "in sleep and said, i have been complaints, they were in the money she said, and the nut.  immediately the chamber, and there in the chamber, and blood, and, and began to repating there she came the nut.  and the bride had been as the bride was again led me, where they seated her where they for sease, but the princess there in the might of them and dragess, and there in the\n",
      "chamber there by that, and there in who had been the bird formed it, there went the\n",
      "princess, who had been complace.  then they sat down and said, i have been prepared\n",
      "themselves and but on the chamber the prince went to the red of the princess, who was perfectly safe and they were in the\n",
      "chamber, and but on the chamber, and said, i have been complaints,\n",
      "and said, i will \n",
      "====================================================================\n",
      "\n",
      "(18).(96).(40).(95).(54).(2).(52).(37).(44).(55).\n",
      "Average loss at step 7: 2.244664\n",
      "\tPerplexity at step 7: 9.437240\n",
      "\n",
      "Valid Perplexity: 21.08\n",
      "\n",
      "Generated Text after epoch 6 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t rself.  then the king said to him.  now the king, and carried to be asked the sack, and i will be free that the king content until than the king, the king, and i will be stopped the other there was in the cook the sack, and they were was a brother that he\n",
      "said, and the sack there is magiUNKin from each other, and carried\n",
      "into the chited, and when they had been them and said, it was carrying there is with them.  then the bridays all the content and the blower saw the six with them.  then the strong one who has before it magiUNKin the blower of the blue sky over it amongst the\n",
      "air.  and the king, and\n",
      "the king and saw that i will blow them, the other the sack to them, but stopped to the strong one white that he said, i will just make the six conveyed the king come to take his comrades.  then the king come into the country, and then they were it was in the chiter and caused it amongst them, and they went to be advice.  there there is magiven awart them and said, i was not down to the strong on\n",
      "====================================================================\n",
      "\n",
      "(13).(94).(42).(75).(66).(28).(98).(35).(23).(88).\n",
      "Average loss at step 8: 2.245995\n",
      "\tPerplexity at step 8: 9.449814\n",
      "\n",
      "Valid Perplexity: 24.00\n",
      "\n",
      "Generated Text after epoch 7 ... \n",
      "======================== New text Segment ==========================\n",
      "\t was soon coming that they have well.\" then he lighUNKing was so much as to done nothing was servant, \"but do not allowed to him, and that the soldier pocket, and she was not had soldier prison.\"\n",
      "then the mannikin was so much as the soldier to his cudgel fell done, and when the morning this way the king was servant, but\n",
      "light to die, the\n",
      "blue light and said, \"what the soldier three,\" said the\n",
      "king, and whosoever was so much as the soldier peased the king, and when his peases the soldier's mercy, and the soldier to the king, \"but it to still for his\n",
      "own, and the blue felf the soldier's, and said, \"when i have you been.  when he threw himself on the stree man came and she was to to drew them in this kingdom for his\n",
      "own, and that when he the soldier, and that if you should take your life.\" then the soldier pulled me as that the soldier to the king was to tain his people on a servant's for his\n",
      "own, and the king, \"but it to him to\n",
      "may smoke only as the mannikin his\n",
      "own, and then the mannikin wa\n",
      "====================================================================\n",
      "\n",
      "(54).(59).(28).(58).(99).(57).(77).(2).(17).(92).\n",
      "Average loss at step 9: 1.728322\n",
      "\tPerplexity at step 9: 5.631196\n",
      "\n",
      "Valid Perplexity: 26.40\n",
      "\n",
      "Generated Text after epoch 8 ... \n",
      "======================== New text Segment ==========================\n",
      "\t she was standing into trees of the dove. and lived\n",
      "happily, and said,\n",
      "\"you have delivered the girl was seemed just as if it was soft and lived\n",
      "happily, and said them forth to her, for he was lived\n",
      "happily and lived\n",
      "happily.  then the branches twined\n",
      "around which had the little white dove, and said,\n",
      "\"you have delivered the ring it to his\n",
      "kingdom, for he was a king's son, and she took the ring, and so long as the room for the power of the tree had\n",
      "tood, she said, \"good-by the dove.\n",
      "as she had a white dove, and said the ring i\n",
      "could not regain my human form.\" then she had changed me into a tree, and went home of her, and said, \"good-day my\n",
      "chouse and every day for two\n",
      "hours i was a white dove, and so long as she came to the door.  \"when\n",
      "the girl was stood and they married, and stood beside him.  and he led them forth to the toom.  and he would come and went to the ring, and said,\n",
      "\"you have determined and\n",
      "looked with it, and she was likewise been changed and kissed her heartily, and said,\n",
      "\"y\n",
      "====================================================================\n",
      "\n",
      "(77).(90).(22).(68).(49).(29).(44).(67).(0).(3).\n",
      "Average loss at step 10: 2.130732\n",
      "\tPerplexity at step 10: 8.421025\n",
      "\n",
      "Valid Perplexity: 22.92\n",
      "\n",
      "Generated Text after epoch 9 ... \n",
      "======================== New text Segment ==========================\n",
      "\t knocked against eaten to drink, the kid had began to the wall, and stooped over the water to drink.  at the well to drink.  but when she was something was now all the wolf had had been stooped over the water to drink, the\n",
      "water to his\n",
      "greated and said, now the walk and saw the wolf and\n",
      "rattled.  then cried, never one cut, there on the stomach one could get in, and said, now her stones\n",
      "thither mother, and had his stomach made him, and had a sight the well and stooped over the water to drink, the kids, and the wolf said, stones.  i thought 'twas still alive, and was mother sewed him fall in, and they were to the wedding and said, there.  when he began to drink, the stones in his fill again, the stones, and the\n",
      "water to his\n",
      "legs and tumbles and tumbles again while he got in, and stooped over the water and put as not the walk and put as many while he was mother mother, and the kids dragged the wolf, and when the still asleep.  then the stones in his stones made him fall in, and they were nev\n",
      "====================================================================\n",
      "\n",
      "(69).(79).(7).(30).(41).(90).(78).(15).(13).(38).\n",
      "Average loss at step 11: 2.209393\n",
      "\tPerplexity at step 11: 9.110188\n",
      "\n",
      "Valid Perplexity: 19.18\n",
      "\n",
      "Generated Text after epoch 10 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  going to the court, and they lived contented to the whole were seized her each other in the maid finished\n",
      "plucking the house, the screamed, that he could not all splendor, and the maid finished\n",
      "plucking the country, the maid was nothing and cooked their heads and the maid finifired the that the maid fired the thorn-house, and the maid finished\n",
      "plucking asleep, and at last the\n",
      "king's son with\n",
      "a kitchen the wall, but the three of the world, and the thorn-ingle\n",
      "of the worn-into the cook of\n",
      "their heads from\n",
      "under their wings.  and the king's son with briar-rose was\n",
      "cried their heads and sizUNKe again, and the king asleep up and was soon the boy went down and the maid finished\n",
      "plucking to the\n",
      "end of the king's son with briar-rose was\n",
      "celebrated with all standing the horses in the king's son come which the country the court still of looked room which when the king's son was\n",
      "celebrated with the boy\n",
      "as the three her into the boy where the boy such\n",
      "a box on the\n",
      "end of their days of the king's hon\n",
      "====================================================================\n",
      "\n",
      "(25).(23).(4).(35).(60).(64).(40).(78).(51).(12).\n",
      "Average loss at step 12: 2.097924\n",
      "\tPerplexity at step 12: 8.149238\n",
      "\n",
      "Valid Perplexity: 20.34\n",
      "\n",
      "Generated Text after epoch 11 ... \n",
      "======================== New text Segment ==========================\n",
      "\t all.  he was thunderstruck, there is the wedding-guests\n",
      "are that you down to life again, and thus them, and sent out to sea, where they sorrow with me in a ship which had gone on his husband.  then she was\n",
      "placed with her knows and begged suddenly ill and died, and the king\n",
      "said, i cannot been the reward that you have murdered his here.  when they both then she was placed with her accomplice in a ship which had been pierced with\n",
      "holence.  then she was\n",
      "placed with her accomplice in his\n",
      "death, and there, you will be the snake-learnt they both there they soon sank amid, and sent out of the great grew out to sea, where they soon sank amid he had been the dead so that the king three girl they deared the great great home and ready to die you the king\n",
      "said, i cannot given me her before the two come with her knees and sent out to sea, where they soon can oves the great ship which had been\n",
      "pierced with him in his\n",
      "sleep, and strikes the doll\n",
      "on his head so that her cap falls off.\n",
      "then she saw the \n",
      "====================================================================\n",
      "\n",
      "(82).(6).(88).(2).(35).(49).(62).(95).(0).(50).\n",
      "Average loss at step 13: 2.161670\n",
      "\tPerplexity at step 13: 8.685632\n",
      "\n",
      "Valid Perplexity: 18.60\n",
      "\n",
      "Generated Text after epoch 12 ... \n",
      "======================== New text Segment ==========================\n",
      "\t iful than the king have golden ring for the corner, and threw it.  she sprang\n",
      "away.  then she was more beautiful, and the king, and had a man lerleiraUNK and took as she could no longer, but she cooked and said, i have contle of joy before in the\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "marriage was solemnized, and and was she asked, and threw her\n",
      "mantle of the king, and when he had gone beautiful, and\n",
      "all the should no longer and she said, but that she cooked to\n",
      "relearnted to\n",
      "released, her golden ring forth, and then he caused\n",
      "her face, and had a man lerleiraUNK had soup that he\n",
      "fetched the cook which he had\n",
      "always but to be look at the beautiful than it was to go up the cook, she was more begroom's son.\n",
      "\n",
      "when she\n",
      "could not like before.  then she had been to the king they was to be are the bride, and that in the\n",
      "cook was so quickly\n",
      "that the king said, you are you she had been so until into her, the king, and where it was so quickly throw her golden\n",
      "said, but as the king, and when he had driven to herself been so quickly, and we \n",
      "====================================================================\n",
      "\n",
      "(67).(74).(44).(79).(20).(57).(47).(32).(29).(72).\n",
      "Average loss at step 14: 2.000718\n",
      "\tPerplexity at step 14: 7.394365\n",
      "\n",
      "Valid Perplexity: 17.92\n",
      "\n",
      "Generated Text after epoch 13 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  my the middle of it, the king's door, and show the prince she right one, and thought of the rode that he\n",
      "was the\n",
      "king's daughter, and said, \"i will go to be that they were to go out, and said the king the king's sun again.\n",
      "\n",
      "then the king's daughter the prince was the kingdom.  and said, \"i have been\n",
      "middle, and said he was sorrow, and when the pardoes came back and said, \"i have not horse had put the forest and the prince was not the right people that he had goned him the door, and the king's bride, and said the right one, and the prince went out of the forest.  they went to the forest of the road, and said, \"i have been turned at one of the father, and his horse who had put out of the middle of it, and\n",
      "thought the\n",
      "princess was now he said, when he had never that they had neverther they said, and told the king's daughter, and should\n",
      "sea, how he\n",
      "had the robber, who had nevertheless was so the princess was strict of the kingdom.\n",
      "\n",
      "at they had put it, and they had have so that he thought, i\n",
      "====================================================================\n",
      "\n",
      "(21).(29).(14).(61).(28).(42).(92).(87).(17).(11).\n",
      "Average loss at step 15: 2.103466\n",
      "\tPerplexity at step 15: 8.194520\n",
      "\n",
      "Valid Perplexity: 18.42\n",
      "\n",
      "Generated Text after epoch 14 ... \n",
      "======================== New text Segment ==========================\n",
      "\t e had to her, and that he had not be gretel was not be all the old woman, who hansel and said the woman, hansel went hansel, when she was still\n",
      "father's house, the door children, they went home\n",
      "and\n",
      "hansel, and they showed the witcUNKs hansel, and that she said they shower of the old woman, when she cried -\n",
      "     the maiden, and hansel, and said, there was still in the well, and all his children hansel threw him.  then the witch,\n",
      "and they were of the witches they had see they make himself, and had not known, and they make her hansel, and they make him, but the\n",
      "woman walked herself to her, and stood beneath it will burn.  they said, and they were out to have happened.  they said the door, and said to her, they went to the gretel was at in the forest to woman.  then the door, said the way of it, and said, and that is not be gretel began to be duck, she was stay and looked into the woman, who is a big kettleful of pitch and hansel, said the old woman, and the wicked to the gretel, and they liv\n",
      "====================================================================\n",
      "\n",
      "(53).(91).(26).(51).(64).(90).(29).(47).(25).(77).\n",
      "Average loss at step 16: 1.886350\n",
      "\tPerplexity at step 16: 6.595254\n",
      "\n",
      "Valid Perplexity: 39.40\n",
      "\n",
      "Generated Text after epoch 15 ... \n",
      "======================== New text Segment ==========================\n",
      "\t there and just said, stop,\n",
      "little pot, and it stopped and gave up cooking, and whosoever wished\n",
      "to return to the town had to stop it.  at last when only one\n",
      "single house remained, the child came home and just said, stop,\n",
      "little pot, and it stopped and gave up cooking, and whosoever wished\n",
      "to return to the town had to eat his way back.  then the\n",
      "next house, and then the whole street, just as if it wanted to\n",
      "satisfy the\n",
      "hunger of the whole world, and there was satisfied, and then\n",
      "she whole house were full, and then the girl had gone out, her mother, and then the greatest\n",
      "distress, but no one knew how to stop it.  once on until the kitchen and there was the greatest\n",
      "distress, but no one knew how to stop it.  at last when only one\n",
      "single house remained, the child came home and just said, stop,\n",
      "little pot, and it stopped and gave up cooking, and whosoever wished\n",
      "to return to the town had to eat his way back.  the girl took, little pot,\n",
      "cook.  and then the\n",
      "next house, and then the whole world,\n",
      "====================================================================\n",
      "\n",
      "(39).(92).(63).(68).(97).(22).(90).(29).(95).(7).\n",
      "Average loss at step 17: 2.052526\n",
      "\tPerplexity at step 17: 7.787549\n",
      "\n",
      "Valid Perplexity: 17.78\n",
      "\n",
      "Generated Text after epoch 16 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  how fares my child, how fares my roe.\n",
      "     the king could be the king cannot because of that, she was seen had been led the door who was delivered him over to the king cannot speak to her.  then he will i come, then never more.\n",
      "and said, you must be to be said, the king the child, and the nurse with the roebuck, my child, and the queen again, and when he had been nurse, and\n",
      "said,\n",
      "     that she saw that the king said,\n",
      "     so the king and that the king come to the world.  the king said to the bath-room, and she nursed it was cast on her daughter had been never more.\n",
      "then she said, ah, god.  but he was so that he had been nurse in his\n",
      "little queen, and said,\n",
      "     how fares my child, how fares my child, and she could not make the child as her daughter was sorrowful.\n",
      "\n",
      "then he had gone again.  then the king was seed, but he was to do the judge, and the nurse, and said, i will was about to be seen the\n",
      "light into the door who was delivered\n",
      "again.  the child as it was\n",
      "torn to be seen that it wa\n",
      "====================================================================\n",
      "\n",
      "(88).(44).(56).(52).(35).(32).(85).(76).(90).(98).\n",
      "Average loss at step 18: 2.016381\n",
      "\tPerplexity at step 18: 7.511097\n",
      "\n",
      "Valid Perplexity: 17.90\n",
      "\n",
      "Generated Text after epoch 17 ... \n",
      "======================== New text Segment ==========================\n",
      "\t forted from their sisters the tree, and the evening them two-eyes, \"i do?\" come to her fruit, and was\n",
      "kind to the tree, however, three-eyes, \"that they went again, and said to her in their faces,\n",
      "and there was solemnized with great beauty, and said to\n",
      "her to the maiden\n",
      "came and said, \"i suffer from it, however, standing them, but he was about their faces, and said, \"you, answered their poor belong, and three-eyes with the tree, and three-eyes, and three-eyes with you shall do?\" cried the knight, \"that i will try it.\" then her and said the knight, anyone three-eye and drops.  and they should not beautiful close to them, so\n",
      "that they were alone, and sat a branch with all the transfortune in their\n",
      "sister, what down and was so much as that the bread\n",
      "of them to such poverty that they had been came and three-eyes, and then two-eyes saw the tree was so that they had to have them with all their bread, and so that they went to the evil two-eyes was\n",
      "kind, and said, \"two-eyes, \"i will go on the goa\n",
      "====================================================================\n",
      "\n",
      "(39).(91).(51).(4).(69).(57).(89).(38).(24).(12).\n",
      "Average loss at step 19: 1.852185\n",
      "\tPerplexity at step 19: 6.373732\n",
      "\n",
      "Valid Perplexity: 19.69\n",
      "\n",
      "Generated Text after epoch 18 ... \n",
      "======================== New text Segment ==========================\n",
      "\t d that the king saw her knees and begged their way, and said, there is to died,\n",
      "and they did not\n",
      "know one woman who was slept, and the waithful service.  the two come out and knew not if he would happened with all that you have murdered with them, and sent at the chamber,\n",
      "and but the way and ready to sea, where he said,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "who had been counteneath the ready to sea, there they soon sank amid a meat that they went to the grave, and strike the others, and sent out to sea, there is your husband.  ah, but you deserve.  they saw her knows and sent out to sea, then he should be\n",
      "his holiness the great ship which had been pierced to life again, and that he had to sing a mass, and said, i will make the reward that you have murdered with their struck, and said, which had been they lived that you deserve.  he was to light, and the two come out to me, and which had been asleep, and shall receive the waves, and they were all.  the king said, i will make you have with\n",
      "me.  he was shut him to his head to l\n",
      "====================================================================\n",
      "\n",
      "(98).(29).(49).(7).(22).(79).(20).(17).(26).(45).\n",
      "Average loss at step 20: 2.010025\n",
      "\tPerplexity at step 20: 7.463507\n",
      "\n",
      "Valid Perplexity: 16.17\n",
      "\n",
      "Generated Text after epoch 19 ... \n",
      "======================== New text Segment ==========================\n",
      "\t let him all that had the fox was not death, and now they came to him as they came with the brothers were stood at the\n",
      "window, and he was to the king's come back to the golden castle, and said, the bird beautiful princess that is not to the king's chambers they came\n",
      "that they\n",
      "wanted to eat them, and then the fox, and the king.  said her went into a wood.  at the king's house.  then the king's son said,\n",
      "you have been so the king's son was not to this house, and the king.  she thought the brothers were to be let to the king's head and left off as they cannot give him and said,\n",
      "yes, i will no other than the golden bird, and they went out to her to the goat's son and will before him to the king's son was she was not done when it was freed\n",
      "from the fox was\n",
      "changed into a king's come back and lived to the king.  and said,\n",
      "you have everything to the king's house, and then they came, they could not have evening\n",
      "of all his beautiful tomb in the fox was\n",
      "changed him and shoot to him, but he was to d\n",
      "====================================================================\n",
      "\n",
      "(88).(92).(2).(17).(4).(15).(79).(89).(52).(16).\n",
      "Average loss at step 21: 1.730540\n",
      "\tPerplexity at step 21: 5.643699\n",
      "\n",
      "Valid Perplexity: 21.02\n",
      "\n",
      "Generated Text after epoch 20 ... \n",
      "======================== New text Segment ==========================\n",
      "\t n this.  he\n",
      "answered, that he\n",
      "would not have\n",
      "discovered it behind her.\n",
      "\n",
      "next morning, the king's daughter\n",
      "wanted to\n",
      "leave it.  how the king's\n",
      "daughter usually wore, and when they had guessed the king's\n",
      "daughter usually wore, and she had guessed her, and\n",
      "this, and she was\n",
      "tole into my meast be\n",
      "answered him, and when the king's\n",
      "daughter usually wore, she\n",
      "inquired further in heard stone.  however, said he said, let the mantle be thought, and they said, let the misty-grey meantime.  but the king's daughter\n",
      "announced that he was\n",
      "not in the raven, which\n",
      "ate of a riddle, and died, they had forced so quiet in her some three mantles brought thither in the raven and she had told the maid\n",
      "came the maid, and the king's daughter thought this.\n",
      "then the judges saw the misty-grey one which the king's\n",
      "daughter usually wore, and said, she had guessed the riddle, and she had guessed her, and when it did not have\n",
      "discovered her by the house, and the mantle be still mother that the king's daughter\n",
      "annother, \n",
      "====================================================================\n",
      "\n",
      "(58).(46).(38).(83).(25).(21).(47).(96).(29).(42).\n",
      "Average loss at step 22: 1.915704\n",
      "\tPerplexity at step 22: 6.791715\n",
      "\n",
      "Valid Perplexity: 17.81\n",
      "\n",
      "Generated Text after epoch 21 ... \n",
      "======================== New text Segment ==========================\n",
      "\t to her for her, and he went, and then she fell in his knapsack, and he went on all his streated\n",
      "herself on the charcoal-burner, and the man went in the town to the horn and said, the whole countrything somether, and\n",
      "delivered him over to justice.  they got out of the hat.  and had it to be another.  that he who were to drive in his\n",
      "hands of this story better had his heart with them, and the king, and the king and the knapsack of the whole countrything soul was asleep, and said the king and great and he would have been to do not\n",
      "one standing as he was still remained to his daughter to him, and had to be death.  and was still his hat he whole caster to the threw the could not\n",
      "put it with all the could have you.  dear wife and the king and the whole country, and had he had been left\n",
      "standing of the whole of the hat.  and\n",
      "delivered him over to just as she had to drive him, and the man, who had been turned him against the child, and the\n",
      "king's daughter to death that, she should do him, and he\n",
      "====================================================================\n",
      "\n",
      "(60).(37).(57).(95).(50).(52).(78).(77).(44).(71).\n",
      "Average loss at step 23: 1.937693\n",
      "\tPerplexity at step 23: 6.942716\n",
      "\n",
      "Valid Perplexity: 19.26\n",
      "\n",
      "Generated Text after epoch 22 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  come to a heard to be prince.\n",
      "\n",
      "\"i am a castle,\n",
      "and then they were\n",
      "burnt, and the daughter married them the bird of the waters, however, death, and then she had grown quite ill and seek his wife\n",
      "out of the waters, and the daughter married the prison smits of father so much that the wise woman, who was gave her some of the water of the old fisherman of the castle, and she\n",
      "became strong and heart with him to the water of the fall into the found with me.\"\n",
      "\n",
      "the fisherman, when\n",
      "the daughter married the old woman took him again on the old woman had said, and strong and the daughter married the old water, and he was still strang too, and who went\n",
      "ing -\n",
      "      they would\n",
      "not have\n",
      "her some of the wand, and\n",
      "held her who was the old woman has no children back with her found the old woman was to be opened and break, and when\n",
      "the daughter\n",
      "gave her the children back with him, and the daughter\n",
      "gave her some of the water was son.\"\n",
      "\n",
      "\"but he had grown quite ill and weak, so the\n",
      "false sisters were\n",
      "burnt, an\n",
      "====================================================================\n",
      "\n",
      "(21).(95).(45).(97).(85).(40).(3).(72).(16).(68).\n",
      "Average loss at step 24: 2.080944\n",
      "\tPerplexity at step 24: 8.012032\n",
      "\n",
      "Valid Perplexity: 16.84\n",
      "\n",
      "Generated Text after epoch 23 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  behind her.  the king, and the fire and said, all whilst he were and\n",
      "said, has he was away from your and saw him, and said, she could have not returned, but they had been a plate for the\n",
      "two still into a ragges that.  he said he, they were to the king's daughter, and he\n",
      "replied, and he was celebrate of a promise, and he was all the stairs the power of his wife was the devil pouring and poured and said, a sword to his finger, and then they were shepherd.\" and the king's daughter, and then he went away and said the\n",
      "true king had returned, and the king's daughter\n",
      "announced that he was celebrated, and that it will answer them,\n",
      "and said, have you in her child, and when he went into the heads off but the servant,\n",
      "and said, i will you have been she found them the fire and pressed upon\n",
      "him, but he sat down to her head, and\n",
      "wept they tried to this the true king had a golden road, and said she told him and cried.  she went to the foot into a something, and then he was celebrated, and\n",
      "said he was \n",
      "====================================================================\n",
      "\n",
      "(89).(92).(74).(71).(52).(36).(30).(3).(6).(96).\n",
      "Average loss at step 25: 1.698287\n",
      "\tPerplexity at step 25: 5.464579\n",
      "\n",
      "Valid Perplexity: 18.05\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 24 ... \n",
      "======================== New text Segment ==========================\n",
      "\t a glass\n",
      "mountain they sailed the box.\"\n",
      "\n",
      "then she was a\n",
      "great castle, they had a great last the wedding was so larger than she was inside, and then she was at last the king's son, and the king's son and the bridegroom's daughter to drink, and the brideggarden.\n",
      "she wanted to the glass\n",
      "mountain they tore it the bridegroom's daught to the three needles, and the bride heard that they were the bridegroom and the king's daughter to the bridegroom and sought and which are that the king's son was to the glass\n",
      "bride, and they were three needles were all, and in it, and the old\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "woman said, \"you are the three needles which was so slept so much larger that he had not to the brought\n",
      "that they were all to be seen, and when\n",
      "you will not aware of the bridegroom's daughter and so that she had a great lengit, and then you are in the bride heard that is the three three bridegroom's\n",
      "son learnt the bridegroom's\n",
      "son was a great lake, they said, \"i am i,\" said the king's bridegroom's\n",
      "son was so sleep in the cas\n",
      "====================================================================\n",
      "\n",
      "(46).(34).(65).(80).(7).(50).(57).(55).(64).(73).\n",
      "Average loss at step 26: 2.218168\n",
      "\tPerplexity at step 26: 9.190480\n",
      "\n",
      "Valid Perplexity: 17.35\n",
      "\n",
      "Generated Text after epoch 25 ... \n",
      "======================== New text Segment ==========================\n",
      "\t ward and went on learning more, and that he has because the money that he had been come to pass, and said, \"you shall the spirit, and the spirit, and as the for it wind, and he said, \"you shall the money that he went back to his father and as the boy, he began to relate, and the son thereupon the most famous doctor in the son thereupon went away to his plaster, and that was left, then the spirit, \"and we will be the spirit, and that he had been consumed the spirit, and the spirit was left, i have got\n",
      "the money that he had a goldsmith\n",
      "that is a goldsmith,\n",
      "who had come to the boy, i must have you\n",
      "forgotsilver, i will soon been come to the boy then the money,\" said the father, \"how\n",
      "have you have you will have not done, and as lived the secretly in the father, i have not do you have got\n",
      "up to her,\" said the sun, i have made such a back to the first was left, i have got up then he wounds the spirit, \"ah, you have not,\" said the father, and the boy then told her and went\n",
      "on live as luck, then \n",
      "====================================================================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Some hyperparameters needed for the training process\n",
    "\n",
    "num_steps = 26\n",
    "steps_per_document = 100\n",
    "docs_per_step = 10\n",
    "valid_summary = 1\n",
    "train_doc_count = num_files\n",
    "\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "# Capture the behavior of train/valid perplexity over time\n",
    "train_perplexity_ot = []\n",
    "valid_perplexity_ot = []\n",
    "\n",
    "# Initializing variables\n",
    "tf.global_variables_initializer().run()\n",
    "print('Initialized Global Variables ')\n",
    "\n",
    "average_loss = 0 # Calculates the average loss ever few steps\n",
    "\n",
    "# We use the first 10 documents that has \n",
    "# more than 10*steps_per_document bigrams for creating the validation dataset\n",
    "\n",
    "# Identify the first 10 documents following the above condition\n",
    "long_doc_ids = []\n",
    "for di in range(num_files):\n",
    "  if len(data_list[di])>10*steps_per_document:\n",
    "    long_doc_ids.append(di)\n",
    "  if len(long_doc_ids)==10:\n",
    "    break\n",
    "    \n",
    "# Generating validation data\n",
    "data_gens = []\n",
    "valid_gens = []\n",
    "for fi in range(num_files):\n",
    "  # Get all the bigrams if the document id is not in the validation document ids\n",
    "  if fi not in long_doc_ids:\n",
    "    data_gens.append(DataGeneratorOHE(data_list[fi],batch_size,num_unrollings))\n",
    "  # if the document is in the validation doc ids, only get up to the \n",
    "  # last steps_per_document bigrams and use the last steps_per_document bigrams as validation data\n",
    "  else:\n",
    "    data_gens.append(DataGeneratorOHE(data_list[fi][:-steps_per_document],batch_size,num_unrollings))\n",
    "    # Defining the validation data generator\n",
    "    valid_gens.append(DataGeneratorOHE(data_list[fi][-steps_per_document:],1,1))\n",
    "\n",
    "feed_dict = {}\n",
    "for step in range(num_steps):\n",
    "    \n",
    "    for di in np.random.permutation(train_doc_count)[:docs_per_step]:            \n",
    "        doc_perplexity = 0\n",
    "        for doc_step_id in range(steps_per_document):\n",
    "            \n",
    "            # Get a set of unrolled batches\n",
    "            u_data, u_labels = data_gens[di].unroll_batches()\n",
    "            \n",
    "            # Populate the feed dict by using each of the data batches\n",
    "            # present in the unrolled data\n",
    "            for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):            \n",
    "                feed_dict[train_inputs[ui]] = dat\n",
    "                feed_dict[train_labels[ui]] = lbl\n",
    "            \n",
    "            # Running the TensorFlow operations\n",
    "            _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp], \n",
    "                                                       feed_dict=feed_dict)\n",
    "            \n",
    "            # Update doc_perpelxity variable\n",
    "            doc_perplexity += step_perplexity\n",
    "            \n",
    "            # Update the average_loss variable\n",
    "            average_loss += step_perplexity\n",
    "        \n",
    "        # shows the training progress\n",
    "        print('(%d).'%di,end='') \n",
    "        \n",
    "        # resetting hidden state after processing a single document\n",
    "        # It's still questionable if this adds value in terms of learning\n",
    "        # One one hand it's intuitive to reset the state when learning a new document\n",
    "        # On the other hand this approach creates a bias for the state to be zero\n",
    "        # We encourage the reader to investigate further the effect of resetting the state\n",
    "        #session.run(reset_train_state) # resetting hidden state for each document\n",
    "        session.run(reset_train_state) # resetting hidden state for each document\n",
    "        \n",
    "    print('')\n",
    "    \n",
    "    \n",
    "    # Generate new samples\n",
    "    if (step+1) % valid_summary == 0:\n",
    "      \n",
    "      # Compute average loss\n",
    "      average_loss = average_loss / (valid_summary*docs_per_step*steps_per_document)\n",
    "      \n",
    "      # Print losses  \n",
    "      print('Average loss at step %d: %f' % (step+1, average_loss))\n",
    "      print('\\tPerplexity at step %d: %f' %(step+1, np.exp(average_loss)))\n",
    "      train_perplexity_ot.append(np.exp(average_loss))\n",
    "        \n",
    "      average_loss = 0 # reset loss\n",
    "      \n",
    "      valid_loss = 0 # reset loss\n",
    "        \n",
    "      # calculate valid perplexity\n",
    "      for v_doc_id in range(10):\n",
    "          # Remember we process things as bigrams\n",
    "          # So need to divide by 2\n",
    "          for v_step in range(steps_per_document//2):\n",
    "            uvalid_data,uvalid_labels = valid_gens[v_doc_id].unroll_batches()        \n",
    "\n",
    "            # Run validation phase related TensorFlow operations       \n",
    "            v_perp = session.run(\n",
    "                valid_perplexity_without_exp,\n",
    "                feed_dict = {valid_inputs:uvalid_data[0],valid_labels: uvalid_labels[0]}\n",
    "            )\n",
    "\n",
    "            valid_loss += v_perp\n",
    "            \n",
    "          session.run(reset_valid_state)\n",
    "      \n",
    "          # Reset validation data generator cursor\n",
    "          valid_gens[v_doc_id].reset_indices()      \n",
    "    \n",
    "      print()\n",
    "      v_perplexity = np.exp(valid_loss/(steps_per_document*10.0//2))\n",
    "      print(\"Valid Perplexity: %.2f\\n\"%v_perplexity)\n",
    "      valid_perplexity_ot.append(v_perplexity)\n",
    "          \n",
    "      decay_learning_rate(session, v_perplexity)\n",
    "\n",
    "      # Generating new text ...\n",
    "      # We will be generating one segment having 500 bigrams\n",
    "      # Feel free to generate several segments by changing\n",
    "      # the value of segments_to_generate\n",
    "      print('Generated Text after epoch %d ... '%step)  \n",
    "      segments_to_generate = 1\n",
    "      chars_in_segment = 500\n",
    "    \n",
    "      for _ in range(segments_to_generate):\n",
    "        print('======================== New text Segment ==========================')\n",
    "        \n",
    "        # Start with a random word\n",
    "        test_word = np.zeros((1,vocabulary_size),dtype=np.float32)\n",
    "        test_word[0,data_list[np.random.randint(0,num_files)][np.random.randint(0,100)]] = 1.0\n",
    "        print(\"\\t\",reverse_dictionary[np.argmax(test_word[0])],end='')\n",
    "        \n",
    "        # Generating words within a segment by feeding in the previous prediction\n",
    "        # as the current input in a recursive manner\n",
    "        for _ in range(chars_in_segment):    \n",
    "          sample_pred = session.run(test_prediction, feed_dict = {test_input:test_word})            \n",
    "          next_ind = sample(sample_pred.ravel())\n",
    "          test_word = np.zeros((1,vocabulary_size),dtype=np.float32)\n",
    "          test_word[0,next_ind] = 1.0\n",
    "          print(reverse_dictionary[next_ind],end='')\n",
    "        print(\"\")\n",
    "        \n",
    "        # Reset train state\n",
    "        session.run(reset_test_state)\n",
    "        print('====================================================================')\n",
    "      print(\"\")\n",
    "\n",
    "session.close()\n",
    "\n",
    "# Write the perplexity data to a CSV\n",
    "\n",
    "with open(filename_to_save, 'wt') as f:\n",
    "    writer = csv.writer(f,delimiter=',')\n",
    "    writer.writerow(train_perplexity_ot)\n",
    "    writer.writerow(valid_perplexity_ot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
